<!DOCTYPE html>
<!-- saved from url=(0046)https://www.zybuluo.com/hanbingtao/note/476663 -->
<html class="theme theme-white"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
    
    
    <meta name="description" content="Cmd Markdown 编辑阅读器，支持实时同步预览，区分写作和阅读模式，支持在线存储，分享文稿网址。">
    <meta name="author" content="Jiawei Zhang">

    <meta name="viewport" content="width=device-width, initial-scale=1.0">

    
    <title>零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器</title>


    <link href="https://www.zybuluo.com/static/img/favicon.png" type="image/x-icon" rel="icon">

    <link href="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/1bc053c8.base.lib.min.css" rel="stylesheet" media="screen">


    
    <!-- id="prettify-style" will be used to get the link element below and change href to change prettify code, so it can't be in beginmin/endmin block. -->
    <link id="prettify-style" href="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/prettify-cmd.css" type="text/css" rel="stylesheet">
    <!--
    <link id="mermaid-style" href="https://www.zybuluo.com/static/editor/libs/mermaid/mermaid.forest.css" type="text/css" rel="stylesheet">
    -->
    <link href="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/45c7d56d.layout.min.css" rel="stylesheet" media="screen">


    

    <script async="" src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/analytics.js"></script><script>
      (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
      })(window,document,'script','//www.google-analytics.com/analytics.js','ga');

      ga('create', 'UA-44461741-1', 'zybuluo.com');
      ga('send', 'pageview');
    </script>
<style type="text/css">.MathJax_Hover_Frame {border-radius: .25em; -webkit-border-radius: .25em; -moz-border-radius: .25em; -khtml-border-radius: .25em; box-shadow: 0px 0px 15px #83A; -webkit-box-shadow: 0px 0px 15px #83A; -moz-box-shadow: 0px 0px 15px #83A; -khtml-box-shadow: 0px 0px 15px #83A; border: 1px solid #A6D ! important; display: inline-block; position: absolute}
.MathJax_Hover_Arrow {position: absolute; width: 15px; height: 11px; cursor: pointer}
</style><style type="text/css">#MathJax_About {position: fixed; left: 50%; width: auto; text-align: center; border: 3px outset; padding: 1em 2em; background-color: #DDDDDD; color: black; cursor: default; font-family: message-box; font-size: 120%; font-style: normal; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; z-index: 201; border-radius: 15px; -webkit-border-radius: 15px; -moz-border-radius: 15px; -khtml-border-radius: 15px; box-shadow: 0px 10px 20px #808080; -webkit-box-shadow: 0px 10px 20px #808080; -moz-box-shadow: 0px 10px 20px #808080; -khtml-box-shadow: 0px 10px 20px #808080; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
.MathJax_Menu {position: absolute; background-color: white; color: black; width: auto; padding: 5px 0px; border: 1px solid #CCCCCC; margin: 0; cursor: default; font: menu; text-align: left; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; z-index: 201; border-radius: 5px; -webkit-border-radius: 5px; -moz-border-radius: 5px; -khtml-border-radius: 5px; box-shadow: 0px 10px 20px #808080; -webkit-box-shadow: 0px 10px 20px #808080; -moz-box-shadow: 0px 10px 20px #808080; -khtml-box-shadow: 0px 10px 20px #808080; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
.MathJax_MenuItem {padding: 1px 2em; background: transparent}
.MathJax_MenuArrow {position: absolute; right: .5em; color: #666666}
.MathJax_MenuActive .MathJax_MenuArrow {color: white}
.MathJax_MenuCheck {position: absolute; left: .7em}
.MathJax_MenuRadioCheck {position: absolute; left: .7em}
.MathJax_MenuLabel {padding: 1px 2em 3px 1.33em; font-style: italic}
.MathJax_MenuRule {border-top: 1px solid #DDDDDD; margin: 4px 3px}
.MathJax_MenuDisabled {color: GrayText}
.MathJax_MenuActive {background-color: #606872; color: white}
.MathJax_Menu_Close {position: absolute; width: 31px; height: 31px; top: -15px; left: -15px}
</style><style type="text/css">#MathJax_Zoom {position: absolute; background-color: #F0F0F0; overflow: auto; display: block; z-index: 301; padding: .5em; border: 1px solid black; margin: 0; font-weight: normal; font-style: normal; text-align: left; text-indent: 0; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; box-shadow: 5px 5px 15px #AAAAAA; -webkit-box-shadow: 5px 5px 15px #AAAAAA; -moz-box-shadow: 5px 5px 15px #AAAAAA; -khtml-box-shadow: 5px 5px 15px #AAAAAA; filter: progid:DXImageTransform.Microsoft.dropshadow(OffX=2, OffY=2, Color='gray', Positive='true')}
#MathJax_ZoomOverlay {position: absolute; left: 0; top: 0; z-index: 300; display: inline-block; width: 100%; height: 100%; border: 0; padding: 0; margin: 0; background-color: white; opacity: 0; filter: alpha(opacity=0)}
#MathJax_ZoomFrame {position: relative; display: inline-block; height: 0; width: 0}
#MathJax_ZoomEventTrap {position: absolute; left: 0; top: 0; z-index: 302; display: inline-block; border: 0; padding: 0; margin: 0; background-color: white; opacity: 0; filter: alpha(opacity=0)}
</style><style type="text/css">.MathJax_Preview {color: #888}
#MathJax_Message {position: fixed; left: 1em; bottom: 1.5em; background-color: #E6E6E6; border: 1px solid #959595; margin: 0px; padding: 2px 8px; z-index: 102; color: black; font-size: 80%; width: auto; white-space: nowrap}
#MathJax_MSIE_Frame {position: absolute; top: 0; left: 0; width: 0px; z-index: 101; border: 0px; margin: 0px; padding: 0px}
.MathJax_Error {color: #CC0000; font-style: italic}
</style><style type="text/css">.MathJax_SVG_Display {text-align: center; margin: 1em 0em; position: relative; display: block; width: 100%}
#MathJax_SVG_Tooltip {background-color: InfoBackground; color: InfoText; border: 1px solid black; box-shadow: 2px 2px 5px #AAAAAA; -webkit-box-shadow: 2px 2px 5px #AAAAAA; -moz-box-shadow: 2px 2px 5px #AAAAAA; -khtml-box-shadow: 2px 2px 5px #AAAAAA; padding: 3px 4px; z-index: 401; position: absolute; left: 0; top: 0; width: auto; height: auto; display: none}
.MathJax_SVG {display: inline; font-style: normal; font-weight: normal; line-height: normal; font-size: 100%; font-size-adjust: none; text-indent: 0; text-align: left; text-transform: none; letter-spacing: normal; word-spacing: normal; word-wrap: normal; white-space: nowrap; float: none; direction: ltr; border: 0; padding: 0; margin: 0}
.MathJax_SVG * {transition: none; -webkit-transition: none; -moz-transition: none; -ms-transition: none; -o-transition: none}
.mjx-svg-href {fill: blue; stroke: blue}
.MathJax_SVG_Processing {visibility: hidden; position: absolute; top: 0; left: 0; width: 0; height: 0; overflow: hidden; display: block}
.MathJax_SVG_Processed {display: none!important}
.MathJax_SVG_ExBox {display: block; overflow: hidden; width: 1px; height: 60ex}
.MathJax_SVG .noError {vertical-align: ; font-size: 90%; text-align: left; color: black; padding: 1px 3px; border: 1px solid}
</style></head>

<body class="theme  theme-white   pace-done"><div class="pace  pace-inactive"><div class="pace-progress" data-progress-text="100%" data-progress="99" style="width: 100%;">
  <div class="pace-progress-inner"></div>
</div>
<div class="pace-activity"></div></div><div style="visibility: hidden; overflow: hidden; position: absolute; top: 0px; height: 1px; width: auto; padding: 0px; border: 0px; margin: 0px; text-align: left; text-indent: 0px; text-transform: none; line-height: normal; letter-spacing: normal; word-spacing: normal;"><div id="MathJax_SVG_Hidden"></div><svg><defs id="MathJax_SVG_glyphs"><path id="MJMATHI-78" stroke-width="1" d="M52 289Q59 331 106 386T222 442Q257 442 286 424T329 379Q371 442 430 442Q467 442 494 420T522 361Q522 332 508 314T481 292T458 288Q439 288 427 299T415 328Q415 374 465 391Q454 404 425 404Q412 404 406 402Q368 386 350 336Q290 115 290 78Q290 50 306 38T341 26Q378 26 414 59T463 140Q466 150 469 151T485 153H489Q504 153 504 145Q504 144 502 134Q486 77 440 33T333 -11Q263 -11 227 52Q186 -10 133 -10H127Q78 -10 57 16T35 71Q35 103 54 123T99 143Q142 143 142 101Q142 81 130 66T107 46T94 41L91 40Q91 39 97 36T113 29T132 26Q168 26 194 71Q203 87 217 139T245 247T261 313Q266 340 266 352Q266 380 251 392T217 404Q177 404 142 372T93 290Q91 281 88 280T72 278H58Q52 284 52 289Z"></path><path id="MJMAIN-20D7" stroke-width="1" d="M-123 694Q-123 702 -118 708T-103 714Q-93 714 -88 706T-80 687T-67 660T-40 633Q-29 626 -29 615Q-29 606 -36 600T-53 590T-83 571T-121 531Q-135 516 -143 516T-157 522T-163 536T-152 559T-129 584T-116 595H-287L-458 596Q-459 597 -461 599T-466 602T-469 607T-471 615Q-471 622 -458 635H-99Q-123 673 -123 694Z"></path><path id="MJMATHI-77" stroke-width="1" d="M580 385Q580 406 599 424T641 443Q659 443 674 425T690 368Q690 339 671 253Q656 197 644 161T609 80T554 12T482 -11Q438 -11 404 5T355 48Q354 47 352 44Q311 -11 252 -11Q226 -11 202 -5T155 14T118 53T104 116Q104 170 138 262T173 379Q173 380 173 381Q173 390 173 393T169 400T158 404H154Q131 404 112 385T82 344T65 302T57 280Q55 278 41 278H27Q21 284 21 287Q21 293 29 315T52 366T96 418T161 441Q204 441 227 416T250 358Q250 340 217 250T184 111Q184 65 205 46T258 26Q301 26 334 87L339 96V119Q339 122 339 128T340 136T341 143T342 152T345 165T348 182T354 206T362 238T373 281Q402 395 406 404Q419 431 449 431Q468 431 475 421T483 402Q483 389 454 274T422 142Q420 131 420 107V100Q420 85 423 71T442 42T487 26Q558 26 600 148Q609 171 620 213T632 273Q632 306 619 325T593 357T580 385Z"></path><path id="MJMAIN-30" stroke-width="1" d="M96 585Q152 666 249 666Q297 666 345 640T423 548Q460 465 460 320Q460 165 417 83Q397 41 362 16T301 -15T250 -22Q224 -22 198 -16T137 16T82 83Q39 165 39 320Q39 494 96 585ZM321 597Q291 629 250 629Q208 629 178 597Q153 571 145 525T137 333Q137 175 145 125T181 46Q209 16 250 16Q290 16 318 46Q347 76 354 130T362 333Q362 478 354 524T321 597Z"></path><path id="MJMATHI-79" stroke-width="1" d="M21 287Q21 301 36 335T84 406T158 442Q199 442 224 419T250 355Q248 336 247 334Q247 331 231 288T198 191T182 105Q182 62 196 45T238 27Q261 27 281 38T312 61T339 94Q339 95 344 114T358 173T377 247Q415 397 419 404Q432 431 462 431Q475 431 483 424T494 412T496 403Q496 390 447 193T391 -23Q363 -106 294 -155T156 -205Q111 -205 77 -183T43 -117Q43 -95 50 -80T69 -58T89 -48T106 -45Q150 -45 150 -87Q150 -107 138 -122T115 -142T102 -147L99 -148Q101 -153 118 -160T152 -167H160Q177 -167 186 -165Q219 -156 247 -127T290 -65T313 -9T321 21L315 17Q309 13 296 6T270 -6Q250 -11 231 -11Q185 -11 150 11T104 82Q103 89 103 113Q103 170 138 262T173 379Q173 380 173 381Q173 390 173 393T169 400T158 404H154Q131 404 112 385T82 344T65 302T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMAIN-3D" stroke-width="1" d="M56 347Q56 360 70 367H707Q722 359 722 347Q722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153Q722 140 707 133H70Q56 140 56 153Z"></path><path id="MJMATHI-73" stroke-width="1" d="M131 289Q131 321 147 354T203 415T300 442Q362 442 390 415T419 355Q419 323 402 308T364 292Q351 292 340 300T328 326Q328 342 337 354T354 372T367 378Q368 378 368 379Q368 382 361 388T336 399T297 405Q249 405 227 379T204 326Q204 301 223 291T278 274T330 259Q396 230 396 163Q396 135 385 107T352 51T289 7T195 -10Q118 -10 86 19T53 87Q53 126 74 143T118 160Q133 160 146 151T160 120Q160 94 142 76T111 58Q109 57 108 57T107 55Q108 52 115 47T146 34T201 27Q237 27 263 38T301 66T318 97T323 122Q323 150 302 164T254 181T195 196T148 231Q131 256 131 289Z"></path><path id="MJMATHI-69" stroke-width="1" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369T98 420T158 442Q197 442 223 419T250 357Q250 340 236 301T196 196T154 83Q149 61 149 51Q149 26 166 26Q175 26 185 29T208 43T235 78T260 137Q263 149 265 151T282 153Q302 153 302 143Q302 135 293 112T268 61T223 11T161 -11Q129 -11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281T56 279T53 278T49 278T41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-67" stroke-width="1" d="M311 43Q296 30 267 15T206 0Q143 0 105 45T66 160Q66 265 143 353T314 442Q361 442 401 394L404 398Q406 401 409 404T418 412T431 419T447 422Q461 422 470 413T480 394Q480 379 423 152T363 -80Q345 -134 286 -169T151 -205Q10 -205 10 -137Q10 -111 28 -91T74 -71Q89 -71 102 -80T116 -111Q116 -121 114 -130T107 -144T99 -154T92 -162L90 -164H91Q101 -167 151 -167Q189 -167 211 -155Q234 -144 254 -122T282 -75Q288 -56 298 -13Q311 35 311 43ZM384 328L380 339Q377 350 375 354T369 368T359 382T346 393T328 402T306 405Q262 405 221 352Q191 313 171 233T151 117Q151 38 213 38Q269 38 323 108L331 118L384 328Z"></path><path id="MJMATHI-6D" stroke-width="1" d="M21 287Q22 293 24 303T36 341T56 388T88 425T132 442T175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q303 442 384 442Q401 442 415 440T441 433T460 423T475 411T485 398T493 385T497 373T500 364T502 357L510 367Q573 442 659 442Q713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145Q857 144 853 130Q845 101 831 73T785 17T716 -10Q669 -10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291L466 157Q433 26 428 16Q415 -11 385 -11Q372 -11 364 -4T353 8T350 18Q350 29 384 161L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 181Q151 335 151 342Q154 357 154 369Q154 405 129 405Q107 405 92 377T69 316T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-6F" stroke-width="1" d="M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z"></path><path id="MJMATHI-64" stroke-width="1" d="M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z"></path><path id="MJMAIN-28" stroke-width="1" d="M94 250Q94 319 104 381T127 488T164 576T202 643T244 695T277 729T302 750H315H319Q333 750 333 741Q333 738 316 720T275 667T226 581T184 443T167 250T184 58T225 -81T274 -167T316 -220T333 -241Q333 -250 318 -250H315H302L274 -226Q180 -141 137 -14T94 250Z"></path><path id="MJMATHI-54" stroke-width="1" d="M40 437Q21 437 21 445Q21 450 37 501T71 602L88 651Q93 669 101 677H569H659Q691 677 697 676T704 667Q704 661 687 553T668 444Q668 437 649 437Q640 437 637 437T631 442L629 445Q629 451 635 490T641 551Q641 586 628 604T573 629Q568 630 515 631Q469 631 457 630T439 622Q438 621 368 343T298 60Q298 48 386 46Q418 46 427 45T436 36Q436 31 433 22Q429 4 424 1L422 0Q419 0 415 0Q410 0 363 1T228 2Q99 2 64 0H49Q43 6 43 9T45 27Q49 40 55 46H83H94Q174 46 189 55Q190 56 191 56Q196 59 201 76T241 233Q258 301 269 344Q339 619 339 625Q339 630 310 630H279Q212 630 191 624Q146 614 121 583T67 467Q60 445 57 441T43 437H40Z"></path><path id="MJMAIN-22C5" stroke-width="1" d="M78 250Q78 274 95 292T138 310Q162 310 180 294T199 251Q199 226 182 208T139 190T96 207T78 250Z"></path><path id="MJMAIN-29" stroke-width="1" d="M60 749L64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12T224 -76T186 -143T145 -194T113 -227T90 -246Q87 -249 86 -250H74Q66 -250 63 -250T58 -247T55 -238Q56 -237 66 -225Q221 -64 221 250T66 725Q56 737 55 738Q55 746 60 749Z"></path><path id="MJMAIN-31" stroke-width="1" d="M213 578L200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641T273 663Q275 666 285 666Q294 666 302 660V361L303 61Q310 54 315 52T339 48T401 46H427V0H416Q395 3 257 3Q121 3 100 0H88V46H114Q136 46 152 46T177 47T193 50T201 52T207 57T213 61V578Z"></path><path id="MJMAIN-2B" stroke-width="1" d="M56 237T56 250T70 270H369V420L370 570Q380 583 389 583Q402 583 409 568V270H707Q722 262 722 250T707 230H409V-68Q401 -82 391 -82H389H387Q375 -82 369 -68V230H70Q56 237 56 250Z"></path><path id="MJMATHI-65" stroke-width="1" d="M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z"></path><path id="MJMAIN-2212" stroke-width="1" d="M84 237T84 250T98 270H679Q694 262 694 250T679 230H98Q84 237 84 250Z"></path><path id="MJMAIN-32" stroke-width="1" d="M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315T301 241Q265 210 201 149L142 93L218 92Q375 92 385 97Q392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19V31Q50 38 56 46T86 81Q115 113 136 137Q145 147 170 174T204 211T233 244T261 278T284 308T305 340T320 369T333 401T340 431T343 464Q343 527 309 573T212 619Q179 619 154 602T119 569T109 550Q109 549 114 549Q132 549 151 535T170 489Q170 464 154 447T109 429Z"></path><path id="MJMAIN-2032" stroke-width="1" d="M79 43Q73 43 52 49T30 61Q30 68 85 293T146 528Q161 560 198 560Q218 560 240 545T262 501Q262 496 260 486Q259 479 173 263T84 45T79 43Z"></path><path id="MJMATHI-66" stroke-width="1" d="M118 -162Q120 -162 124 -164T135 -167T147 -168Q160 -168 171 -155T187 -126Q197 -99 221 27T267 267T289 382V385H242Q195 385 192 387Q188 390 188 397L195 425Q197 430 203 430T250 431Q298 431 298 432Q298 434 307 482T319 540Q356 705 465 705Q502 703 526 683T550 630Q550 594 529 578T487 561Q443 561 443 603Q443 622 454 636T478 657L487 662Q471 668 457 668Q445 668 434 658T419 630Q412 601 403 552T387 469T380 433Q380 431 435 431Q480 431 487 430T498 424Q499 420 496 407T491 391Q489 386 482 386T428 385H372L349 263Q301 15 282 -47Q255 -132 212 -173Q175 -205 139 -205Q107 -205 81 -186T55 -132Q55 -95 76 -78T118 -61Q162 -61 162 -103Q162 -122 151 -136T127 -157L118 -162Z"></path><path id="MJMATHI-6E" stroke-width="1" d="M21 287Q22 293 24 303T36 341T56 388T89 425T135 442Q171 442 195 424T225 390T231 369Q231 367 232 367L243 378Q304 442 382 442Q436 442 469 415T503 336T465 179T427 52Q427 26 444 26Q450 26 453 27Q482 32 505 65T540 145Q542 153 560 153Q580 153 580 145Q580 144 576 130Q568 101 554 73T508 17T439 -10Q392 -10 371 17T350 73Q350 92 386 193T423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 180T152 343Q153 348 153 366Q153 405 129 405Q91 405 66 305Q60 285 60 284Q58 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-74" stroke-width="1" d="M26 385Q19 392 19 395Q19 399 22 411T27 425Q29 430 36 430T87 431H140L159 511Q162 522 166 540T173 566T179 586T187 603T197 615T211 624T229 626Q247 625 254 615T261 596Q261 589 252 549T232 470L222 433Q222 431 272 431H323Q330 424 330 420Q330 398 317 385H210L174 240Q135 80 135 68Q135 26 162 26Q197 26 230 60T283 144Q285 150 288 151T303 153H307Q322 153 322 145Q322 142 319 133Q314 117 301 95T267 48T216 6T155 -11Q125 -11 98 4T59 56Q57 64 57 83V101L92 241Q127 382 128 383Q128 385 77 385H26Z"></path><path id="MJMATHI-72" stroke-width="1" d="M21 287Q22 290 23 295T28 317T38 348T53 381T73 411T99 433T132 442Q161 442 183 430T214 408T225 388Q227 382 228 382T236 389Q284 441 347 441H350Q398 441 422 400Q430 381 430 363Q430 333 417 315T391 292T366 288Q346 288 334 299T322 328Q322 376 378 392Q356 405 342 405Q286 405 239 331Q229 315 224 298T190 165Q156 25 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 114 189T154 366Q154 405 128 405Q107 405 92 377T68 316T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-6B" stroke-width="1" d="M121 647Q121 657 125 670T137 683Q138 683 209 688T282 694Q294 694 294 686Q294 679 244 477Q194 279 194 272Q213 282 223 291Q247 309 292 354T362 415Q402 442 438 442Q468 442 485 423T503 369Q503 344 496 327T477 302T456 291T438 288Q418 288 406 299T394 328Q394 353 410 369T442 390L458 393Q446 405 434 405H430Q398 402 367 380T294 316T228 255Q230 254 243 252T267 246T293 238T320 224T342 206T359 180T365 147Q365 130 360 106T354 66Q354 26 381 26Q429 26 459 145Q461 153 479 153H483Q499 153 499 144Q499 139 496 130Q455 -11 378 -11Q333 -11 305 15T277 90Q277 108 280 121T283 145Q283 167 269 183T234 206T200 217T182 220H180Q168 178 159 139T145 81T136 44T129 20T122 7T111 -2Q98 -11 83 -11Q66 -11 57 -1T48 16Q48 26 85 176T158 471L195 616Q196 629 188 632T149 637H144Q134 637 131 637T124 640T121 647Z"></path><path id="MJMAIN-34" stroke-width="1" d="M462 0Q444 3 333 3Q217 3 199 0H190V46H221Q241 46 248 46T265 48T279 53T286 61Q287 63 287 115V165H28V211L179 442Q332 674 334 675Q336 677 355 677H373L379 671V211H471V165H379V114Q379 73 379 66T385 54Q393 47 442 46H471V0H462ZM293 211V545L74 212L183 211H293Z"></path><path id="MJMAIN-2C" stroke-width="1" d="M78 35T78 60T94 103T137 121Q165 121 187 96T210 8Q210 -27 201 -60T180 -117T154 -158T130 -185T117 -194Q113 -194 104 -185T95 -172Q95 -168 106 -156T131 -126T157 -76T173 -3V9L172 8Q170 7 167 6T161 3T152 1T140 0Q113 0 96 17Z"></path><path id="MJMAIN-33" stroke-width="1" d="M127 463Q100 463 85 480T69 524Q69 579 117 622T233 665Q268 665 277 664Q351 652 390 611T430 522Q430 470 396 421T302 350L299 348Q299 347 308 345T337 336T375 315Q457 262 457 175Q457 96 395 37T238 -22Q158 -22 100 21T42 130Q42 158 60 175T105 193Q133 193 151 175T169 130Q169 119 166 110T159 94T148 82T136 74T126 70T118 67L114 66Q165 21 238 21Q293 21 321 74Q338 107 338 175V195Q338 290 274 322Q259 328 213 329L171 330L168 332Q166 335 166 348Q166 366 174 366Q202 366 232 371Q266 376 294 413T322 525V533Q322 590 287 612Q265 626 240 626Q208 626 181 615T143 592T132 580H135Q138 579 143 578T153 573T165 566T175 555T183 540T186 520Q186 498 172 481T127 463Z"></path><path id="MJMATHI-61" stroke-width="1" d="M33 157Q33 258 109 349T280 441Q331 441 370 392Q386 422 416 422Q429 422 439 414T449 394Q449 381 412 234T374 68Q374 43 381 35T402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487Q506 153 506 144Q506 138 501 117T481 63T449 13Q436 0 417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157ZM351 328Q351 334 346 350T323 385T277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q217 26 254 59T298 110Q300 114 325 217T351 328Z"></path><path id="MJMATHI-62" stroke-width="1" d="M73 647Q73 657 77 670T89 683Q90 683 161 688T234 694Q246 694 246 685T212 542Q204 508 195 472T180 418L176 399Q176 396 182 402Q231 442 283 442Q345 442 383 396T422 280Q422 169 343 79T173 -11Q123 -11 82 27T40 150V159Q40 180 48 217T97 414Q147 611 147 623T109 637Q104 637 101 637H96Q86 637 83 637T76 640T73 647ZM336 325V331Q336 405 275 405Q258 405 240 397T207 376T181 352T163 330L157 322L136 236Q114 150 114 114Q114 66 138 42Q154 26 178 26Q211 26 245 58Q270 81 285 114T318 219Q336 291 336 325Z"></path><path id="MJMATHI-6A" stroke-width="1" d="M297 596Q297 627 318 644T361 661Q378 661 389 651T403 623Q403 595 384 576T340 557Q322 557 310 567T297 596ZM288 376Q288 405 262 405Q240 405 220 393T185 362T161 325T144 293L137 279Q135 278 121 278H107Q101 284 101 286T105 299Q126 348 164 391T252 441Q253 441 260 441T272 442Q296 441 316 432Q341 418 354 401T367 348V332L318 133Q267 -67 264 -75Q246 -125 194 -164T75 -204Q25 -204 7 -183T-12 -137Q-12 -110 7 -91T53 -71Q70 -71 82 -81T95 -112Q95 -148 63 -167Q69 -168 77 -168Q111 -168 139 -140T182 -74L193 -32Q204 11 219 72T251 197T278 308T289 365Q289 372 288 376Z"></path><path id="MJMAIN-35" stroke-width="1" d="M164 157Q164 133 148 117T109 101H102Q148 22 224 22Q294 22 326 82Q345 115 345 210Q345 313 318 349Q292 382 260 382H254Q176 382 136 314Q132 307 129 306T114 304Q97 304 95 310Q93 314 93 485V614Q93 664 98 664Q100 666 102 666Q103 666 123 658T178 642T253 634Q324 634 389 662Q397 666 402 666Q410 666 410 648V635Q328 538 205 538Q174 538 149 544L139 546V374Q158 388 169 396T205 412T256 420Q337 420 393 355T449 201Q449 109 385 44T229 -22Q148 -22 99 32T50 154Q50 178 61 192T84 210T107 214Q132 214 148 197T164 157Z"></path><path id="MJMAIN-36" stroke-width="1" d="M42 313Q42 476 123 571T303 666Q372 666 402 630T432 550Q432 525 418 510T379 495Q356 495 341 509T326 548Q326 592 373 601Q351 623 311 626Q240 626 194 566Q147 500 147 364L148 360Q153 366 156 373Q197 433 263 433H267Q313 433 348 414Q372 400 396 374T435 317Q456 268 456 210V192Q456 169 451 149Q440 90 387 34T253 -22Q225 -22 199 -14T143 16T92 75T56 172T42 313ZM257 397Q227 397 205 380T171 335T154 278T148 216Q148 133 160 97T198 39Q222 21 251 21Q302 21 329 59Q342 77 347 104T352 209Q352 289 347 316T329 361Q302 397 257 397Z"></path><path id="MJMAIN-37" stroke-width="1" d="M55 458Q56 460 72 567L88 674Q88 676 108 676H128V672Q128 662 143 655T195 646T364 644H485V605L417 512Q408 500 387 472T360 435T339 403T319 367T305 330T292 284T284 230T278 162T275 80Q275 66 275 52T274 28V19Q270 2 255 -10T221 -22Q210 -22 200 -19T179 0T168 40Q168 198 265 368Q285 400 349 489L395 552H302Q128 552 119 546Q113 543 108 522T98 479L95 458V455H55V458Z"></path><path id="MJMAIN-38" stroke-width="1" d="M70 417T70 494T124 618T248 666Q319 666 374 624T429 515Q429 485 418 459T392 417T361 389T335 371T324 363L338 354Q352 344 366 334T382 323Q457 264 457 174Q457 95 399 37T249 -22Q159 -22 101 29T43 155Q43 263 172 335L154 348Q133 361 127 368Q70 417 70 494ZM286 386L292 390Q298 394 301 396T311 403T323 413T334 425T345 438T355 454T364 471T369 491T371 513Q371 556 342 586T275 624Q268 625 242 625Q201 625 165 599T128 534Q128 511 141 492T167 463T217 431Q224 426 228 424L286 386ZM250 21Q308 21 350 55T392 137Q392 154 387 169T375 194T353 216T330 234T301 253T274 270Q260 279 244 289T218 306L210 311Q204 311 181 294T133 239T107 157Q107 98 150 60T250 21Z"></path><path id="MJMAIN-5B" stroke-width="1" d="M118 -250V750H255V710H158V-210H255V-250H118Z"></path><path id="MJMAIN-5D" stroke-width="1" d="M22 710V750H159V-250H22V-210H119V710H22Z"></path><path id="MJSZ4-23A1" stroke-width="1" d="M319 -645V1154H666V1070H403V-645H319Z"></path><path id="MJSZ4-23A3" stroke-width="1" d="M319 -644V1155H403V-560H666V-644H319Z"></path><path id="MJSZ4-23A4" stroke-width="1" d="M0 1070V1154H347V-645H263V1070H0Z"></path><path id="MJSZ4-23A6" stroke-width="1" d="M263 -560V1155H347V-644H0V-560H263Z"></path><path id="MJSZ3-5B" stroke-width="1" d="M247 -949V1450H516V1388H309V-887H516V-949H247Z"></path><path id="MJSZ3-5D" stroke-width="1" d="M11 1388V1450H280V-949H11V-887H218V1388H11Z"></path><path id="MJMAIN-2192" stroke-width="1" d="M56 237T56 250T70 270H835Q719 357 692 493Q692 494 692 496T691 499Q691 511 708 511H711Q720 511 723 510T729 506T732 497T735 481T743 456Q765 389 816 336T935 261Q944 258 944 250Q944 244 939 241T915 231T877 212Q836 186 806 152T761 85T740 35T732 4Q730 -6 727 -8T711 -11Q691 -11 691 0Q691 7 696 25Q728 151 835 230H70Q56 237 56 250Z"></path><path id="MJSZ4-23A2" stroke-width="1" d="M319 0V602H403V0H319Z"></path><path id="MJSZ4-23A5" stroke-width="1" d="M263 0V602H347V0H263Z"></path><path id="MJMAIN-39" stroke-width="1" d="M352 287Q304 211 232 211Q154 211 104 270T44 396Q42 412 42 436V444Q42 537 111 606Q171 666 243 666Q245 666 249 666T257 665H261Q273 665 286 663T323 651T370 619T413 560Q456 472 456 334Q456 194 396 97Q361 41 312 10T208 -22Q147 -22 108 7T68 93T121 149Q143 149 158 135T173 96Q173 78 164 65T148 49T135 44L131 43Q131 41 138 37T164 27T206 22H212Q272 22 313 86Q352 142 352 280V287ZM244 248Q292 248 321 297T351 430Q351 508 343 542Q341 552 337 562T323 588T293 615T246 625Q208 625 181 598Q160 576 154 546T147 441Q147 358 152 329T172 282Q197 248 244 248Z"></path><path id="MJMATHI-57" stroke-width="1" d="M436 683Q450 683 486 682T553 680Q604 680 638 681T677 682Q695 682 695 674Q695 670 692 659Q687 641 683 639T661 637Q636 636 621 632T600 624T597 615Q597 603 613 377T629 138L631 141Q633 144 637 151T649 170T666 200T690 241T720 295T759 362Q863 546 877 572T892 604Q892 619 873 628T831 637Q817 637 817 647Q817 650 819 660Q823 676 825 679T839 682Q842 682 856 682T895 682T949 681Q1015 681 1034 683Q1048 683 1048 672Q1048 666 1045 655T1038 640T1028 637Q1006 637 988 631T958 617T939 600T927 584L923 578L754 282Q586 -14 585 -15Q579 -22 561 -22Q546 -22 542 -17Q539 -14 523 229T506 480L494 462Q472 425 366 239Q222 -13 220 -15T215 -19Q210 -22 197 -22Q178 -22 176 -15Q176 -12 154 304T131 622Q129 631 121 633T82 637H58Q51 644 51 648Q52 671 64 683H76Q118 680 176 680Q301 680 313 683H323Q329 677 329 674T327 656Q322 641 318 637H297Q236 634 232 620Q262 160 266 136L501 550L499 587Q496 629 489 632Q483 636 447 637Q428 637 422 639T416 648Q416 650 418 660Q419 664 420 669T421 676T424 680T428 682T436 683Z"></path><path id="MJMAIN-2E" stroke-width="1" d="M78 60Q78 84 95 102T138 120Q162 120 180 104T199 61Q199 36 182 18T139 0T96 17T78 60Z"></path><path id="MJMATHI-3B4" stroke-width="1" d="M195 609Q195 656 227 686T302 717Q319 716 351 709T407 697T433 690Q451 682 451 662Q451 644 438 628T403 612Q382 612 348 641T288 671T249 657T235 628Q235 584 334 463Q401 379 401 292Q401 169 340 80T205 -10H198Q127 -10 83 36T36 153Q36 286 151 382Q191 413 252 434Q252 435 245 449T230 481T214 521T201 566T195 609ZM112 130Q112 83 136 55T204 27Q233 27 256 51T291 111T309 178T316 232Q316 267 309 298T295 344T269 400L259 396Q215 381 183 342T137 256T118 179T112 130Z"></path><path id="MJSZ2-2211" stroke-width="1" d="M60 948Q63 950 665 950H1267L1325 815Q1384 677 1388 669H1348L1341 683Q1320 724 1285 761Q1235 809 1174 838T1033 881T882 898T699 902H574H543H251L259 891Q722 258 724 252Q725 250 724 246Q721 243 460 -56L196 -356Q196 -357 407 -357Q459 -357 548 -357T676 -358Q812 -358 896 -353T1063 -332T1204 -283T1307 -196Q1328 -170 1348 -124H1388Q1388 -125 1381 -145T1356 -210T1325 -294L1267 -449L666 -450Q64 -450 61 -448Q55 -446 55 -439Q55 -437 57 -433L590 177Q590 178 557 222T452 366T322 544L56 909L55 924Q55 945 60 948Z"></path><path id="MJMAIN-2208" stroke-width="1" d="M84 250Q84 372 166 450T360 539Q361 539 377 539T419 540T469 540H568Q583 532 583 520Q583 511 570 501L466 500Q355 499 329 494Q280 482 242 458T183 409T147 354T129 306T124 272V270H568Q583 262 583 250T568 230H124V228Q124 207 134 177T167 112T231 48T328 7Q355 1 466 0H570Q583 -10 583 -20Q583 -32 568 -40H471Q464 -40 446 -40T417 -41Q262 -41 172 45Q84 127 84 250Z"></path><path id="MJMATHI-75" stroke-width="1" d="M21 287Q21 295 30 318T55 370T99 420T158 442Q204 442 227 417T250 358Q250 340 216 246T182 105Q182 62 196 45T238 27T291 44T328 78L339 95Q341 99 377 247Q407 367 413 387T427 416Q444 431 463 431Q480 431 488 421T496 402L420 84Q419 79 419 68Q419 43 426 35T447 26Q469 29 482 57T512 145Q514 153 532 153Q551 153 551 144Q550 139 549 130T540 98T523 55T498 17T462 -8Q454 -10 438 -10Q372 -10 347 46Q345 45 336 36T318 21T296 6T267 -6T233 -11Q189 -11 155 7Q103 38 103 113Q103 170 138 262T173 379Q173 380 173 381Q173 390 173 393T169 400T158 404H154Q131 404 112 385T82 344T65 302T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-70" stroke-width="1" d="M23 287Q24 290 25 295T30 317T40 348T55 381T75 411T101 433T134 442Q209 442 230 378L240 387Q302 442 358 442Q423 442 460 395T497 281Q497 173 421 82T249 -10Q227 -10 210 -4Q199 1 187 11T168 28L161 36Q160 35 139 -51T118 -138Q118 -144 126 -145T163 -148H188Q194 -155 194 -157T191 -175Q188 -187 185 -190T172 -194Q170 -194 161 -194T127 -193T65 -192Q-5 -192 -24 -194H-32Q-39 -187 -39 -183Q-37 -156 -26 -148H-6Q28 -147 33 -136Q36 -130 94 103T155 350Q156 355 156 364Q156 405 131 405Q109 405 94 377T71 316T59 280Q57 278 43 278H29Q23 284 23 287ZM178 102Q200 26 252 26Q282 26 310 49T356 107Q374 141 392 215T411 325V331Q411 405 350 405Q339 405 328 402T306 393T286 380T269 365T254 350T243 336T235 326L232 322Q232 321 229 308T218 264T204 212Q178 106 178 102Z"></path><path id="MJMAIN-2190" stroke-width="1" d="M944 261T944 250T929 230H165Q167 228 182 216T211 189T244 152T277 96T303 25Q308 7 308 0Q308 -11 288 -11Q281 -11 278 -11T272 -7T267 2T263 21Q245 94 195 151T73 236Q58 242 55 247Q55 254 59 257T73 264Q121 283 158 314T215 375T247 434T264 480L267 497Q269 503 270 505T275 509T288 511Q308 511 308 500Q308 493 303 475Q293 438 278 406T246 352T215 315T185 287T165 270H929Q944 261 944 250Z"></path><path id="MJMATHI-3B7" stroke-width="1" d="M21 287Q22 290 23 295T28 317T38 348T53 381T73 411T99 433T132 442Q156 442 175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q304 442 382 442Q436 442 469 415T503 336V326Q503 302 439 53Q381 -182 377 -189Q364 -216 332 -216Q319 -216 310 -208T299 -186Q299 -177 358 57L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 114 189T154 366Q154 405 128 405Q107 405 92 377T68 316T57 280Q55 278 41 278H27Q21 284 21 287Z"></path><path id="MJMATHI-45" stroke-width="1" d="M492 213Q472 213 472 226Q472 230 477 250T482 285Q482 316 461 323T364 330H312Q311 328 277 192T243 52Q243 48 254 48T334 46Q428 46 458 48T518 61Q567 77 599 117T670 248Q680 270 683 272Q690 274 698 274Q718 274 718 261Q613 7 608 2Q605 0 322 0H133Q31 0 31 11Q31 13 34 25Q38 41 42 43T65 46Q92 46 125 49Q139 52 144 61Q146 66 215 342T285 622Q285 629 281 629Q273 632 228 634H197Q191 640 191 642T193 659Q197 676 203 680H757Q764 676 764 669Q764 664 751 557T737 447Q735 440 717 440H705Q698 445 698 453L701 476Q704 500 704 528Q704 558 697 578T678 609T643 625T596 632T532 634H485Q397 633 392 631Q388 629 386 622Q385 619 355 499T324 377Q347 376 372 376H398Q464 376 489 391T534 472Q538 488 540 490T557 493Q562 493 565 493T570 492T572 491T574 487T577 483L544 351Q511 218 508 216Q505 213 492 213Z"></path><path id="MJMAIN-2261" stroke-width="1" d="M56 444Q56 457 70 464H707Q722 456 722 444Q722 430 706 424H72Q56 429 56 444ZM56 237T56 250T70 270H707Q722 262 722 250T707 230H70Q56 237 56 250ZM56 56Q56 71 72 76H706Q722 70 722 56Q722 44 707 36H70Q56 43 56 56Z"></path><path id="MJMAIN-2202" stroke-width="1" d="M202 508Q179 508 169 520T158 547Q158 557 164 577T185 624T230 675T301 710L333 715H345Q378 715 384 714Q447 703 489 661T549 568T566 457Q566 362 519 240T402 53Q321 -22 223 -22Q123 -22 73 56Q42 102 42 148V159Q42 276 129 370T322 465Q383 465 414 434T455 367L458 378Q478 461 478 515Q478 603 437 639T344 676Q266 676 223 612Q264 606 264 572Q264 547 246 528T202 508ZM430 306Q430 372 401 400T333 428Q270 428 222 382Q197 354 183 323T150 221Q132 149 132 116Q132 21 232 21Q244 21 250 22Q327 35 374 112Q389 137 409 196T430 306Z"></path><path id="MJSZ1-2211" stroke-width="1" d="M61 748Q64 750 489 750H913L954 640Q965 609 976 579T993 533T999 516H979L959 517Q936 579 886 621T777 682Q724 700 655 705T436 710H319Q183 710 183 709Q186 706 348 484T511 259Q517 250 513 244L490 216Q466 188 420 134T330 27L149 -187Q149 -188 362 -188Q388 -188 436 -188T506 -189Q679 -189 778 -162T936 -43Q946 -27 959 6H999L913 -249L489 -250Q65 -250 62 -248Q56 -246 56 -239Q56 -234 118 -161Q186 -81 245 -11L428 206Q428 207 242 462L57 717L56 728Q56 744 61 748Z"></path><path id="MJMATHI-44" stroke-width="1" d="M287 628Q287 635 230 637Q207 637 200 638T193 647Q193 655 197 667T204 682Q206 683 403 683Q570 682 590 682T630 676Q702 659 752 597T803 431Q803 275 696 151T444 3L430 1L236 0H125H72Q48 0 41 2T33 11Q33 13 36 25Q40 41 44 43T67 46Q94 46 127 49Q141 52 146 61Q149 65 218 339T287 628ZM703 469Q703 507 692 537T666 584T629 613T590 629T555 636Q553 636 541 636T512 636T479 637H436Q392 637 386 627Q384 623 313 339T242 52Q242 48 253 48T330 47Q335 47 349 47T373 46Q499 46 581 128Q617 164 640 212T683 339T703 469Z"></path><path id="MJMATHI-3B8" stroke-width="1" d="M35 200Q35 302 74 415T180 610T319 704Q320 704 327 704T339 705Q393 701 423 656Q462 596 462 495Q462 380 417 261T302 66T168 -10H161Q125 -10 99 10T60 63T41 130T35 200ZM383 566Q383 668 330 668Q294 668 260 623T204 521T170 421T157 371Q206 370 254 370L351 371Q352 372 359 404T375 484T383 566ZM113 132Q113 26 166 26Q181 26 198 36T239 74T287 161T335 307L340 324H145Q145 321 136 286T120 208T113 132Z"></path><path id="MJMAIN-6C" stroke-width="1" d="M42 46H56Q95 46 103 60V68Q103 77 103 91T103 124T104 167T104 217T104 272T104 329Q104 366 104 407T104 482T104 542T103 586T103 603Q100 622 89 628T44 637H26V660Q26 683 28 683L38 684Q48 685 67 686T104 688Q121 689 141 690T171 693T182 694H185V379Q185 62 186 60Q190 52 198 49Q219 46 247 46H263V0H255L232 1Q209 2 183 2T145 3T107 3T57 1L34 0H26V46H42Z"></path><path id="MJMAIN-69" stroke-width="1" d="M69 609Q69 637 87 653T131 669Q154 667 171 652T188 609Q188 579 171 564T129 549Q104 549 87 564T69 609ZM247 0Q232 3 143 3Q132 3 106 3T56 1L34 0H26V46H42Q70 46 91 49Q100 53 102 60T104 102V205V293Q104 345 102 359T88 378Q74 385 41 385H30V408Q30 431 32 431L42 432Q52 433 70 434T106 436Q123 437 142 438T171 441T182 442H185V62Q190 52 197 50T232 46H255V0H247Z"></path><path id="MJMAIN-6D" stroke-width="1" d="M41 46H55Q94 46 102 60V68Q102 77 102 91T102 122T103 161T103 203Q103 234 103 269T102 328V351Q99 370 88 376T43 385H25V408Q25 431 27 431L37 432Q47 433 65 434T102 436Q119 437 138 438T167 441T178 442H181V402Q181 364 182 364T187 369T199 384T218 402T247 421T285 437Q305 442 336 442Q351 442 364 440T387 434T406 426T421 417T432 406T441 395T448 384T452 374T455 366L457 361L460 365Q463 369 466 373T475 384T488 397T503 410T523 422T546 432T572 439T603 442Q729 442 740 329Q741 322 741 190V104Q741 66 743 59T754 49Q775 46 803 46H819V0H811L788 1Q764 2 737 2T699 3Q596 3 587 0H579V46H595Q656 46 656 62Q657 64 657 200Q656 335 655 343Q649 371 635 385T611 402T585 404Q540 404 506 370Q479 343 472 315T464 232V168V108Q464 78 465 68T468 55T477 49Q498 46 526 46H542V0H534L510 1Q487 2 460 2T422 3Q319 3 310 0H302V46H318Q379 46 379 62Q380 64 380 200Q379 335 378 343Q372 371 358 385T334 402T308 404Q263 404 229 370Q202 343 195 315T187 232V168V108Q187 78 188 68T191 55T200 49Q221 46 249 46H265V0H257L234 1Q210 2 183 2T145 3Q42 3 33 0H25V46H41Z"></path><path id="MJMATHI-3F5" stroke-width="1" d="M227 -11Q149 -11 95 41T40 174Q40 262 87 322Q121 367 173 396T287 430Q289 431 329 431H367Q382 426 382 411Q382 385 341 385H325H312Q191 385 154 277L150 265H327Q340 256 340 246Q340 228 320 219H138V217Q128 187 128 143Q128 77 160 52T231 26Q258 26 284 36T326 57T343 68Q350 68 354 58T358 39Q358 36 357 35Q354 31 337 21T289 0T227 -11Z"></path><path id="MJMAIN-3E" stroke-width="1" d="M84 520Q84 528 88 533T96 539L99 540Q106 540 253 471T544 334L687 265Q694 260 694 250T687 235Q685 233 395 96L107 -40H101Q83 -38 83 -20Q83 -19 83 -17Q82 -10 98 -1Q117 9 248 71Q326 108 378 132L626 250L378 368Q90 504 86 509Q84 513 84 520Z"></path><path id="MJMAIN-2248" stroke-width="1" d="M55 319Q55 360 72 393T114 444T163 472T205 482Q207 482 213 482T223 483Q262 483 296 468T393 413L443 381Q502 346 553 346Q609 346 649 375T694 454Q694 465 698 474T708 483Q722 483 722 452Q722 386 675 338T555 289Q514 289 468 310T388 357T308 404T224 426Q164 426 125 393T83 318Q81 289 69 289Q55 289 55 319ZM55 85Q55 126 72 159T114 210T163 238T205 248Q207 248 213 248T223 249Q262 249 296 234T393 179L443 147Q502 112 553 112Q609 112 649 141T694 220Q694 249 708 249T722 217Q722 153 675 104T555 55Q514 55 468 76T388 123T308 170T224 192Q164 192 125 159T83 84Q80 55 69 55Q55 55 55 85Z"></path><path id="MJMATHI-6C" stroke-width="1" d="M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z"></path><path id="MJMAIN-221A" stroke-width="1" d="M95 178Q89 178 81 186T72 200T103 230T169 280T207 309Q209 311 212 311H213Q219 311 227 294T281 177Q300 134 312 108L397 -77Q398 -77 501 136T707 565T814 786Q820 800 834 800Q841 800 846 794T853 782V776L620 293L385 -193Q381 -200 366 -200Q357 -200 354 -197Q352 -195 256 15L160 225L144 214Q129 202 113 190T95 178Z"></path><path id="MJMATHI-3B1" stroke-width="1" d="M34 156Q34 270 120 356T309 442Q379 442 421 402T478 304Q484 275 485 237V208Q534 282 560 374Q564 388 566 390T582 393Q603 393 603 385Q603 376 594 346T558 261T497 161L486 147L487 123Q489 67 495 47T514 26Q528 28 540 37T557 60Q559 67 562 68T577 70Q597 70 597 62Q597 56 591 43Q579 19 556 5T512 -10H505Q438 -10 414 62L411 69L400 61Q390 53 370 41T325 18T267 -2T203 -11Q124 -11 79 39T34 156ZM208 26Q257 26 306 47T379 90L403 112Q401 255 396 290Q382 405 304 405Q235 405 183 332Q156 292 139 224T121 120Q121 71 146 49T208 26Z"></path><path id="MJMAIN-3A" stroke-width="1" d="M78 370Q78 394 95 412T138 430Q162 430 180 414T199 371Q199 346 182 328T139 310T96 327T78 370ZM78 60Q78 84 95 102T138 120Q162 120 180 104T199 61Q199 36 182 18T139 0T96 17T78 60Z"></path><path id="MJMAIN-2217" stroke-width="1" d="M229 286Q216 420 216 436Q216 454 240 464Q241 464 245 464T251 465Q263 464 273 456T283 436Q283 419 277 356T270 286L328 328Q384 369 389 372T399 375Q412 375 423 365T435 338Q435 325 425 315Q420 312 357 282T289 250L355 219L425 184Q434 175 434 161Q434 146 425 136T401 125Q393 125 383 131T328 171L270 213Q283 79 283 63Q283 53 276 44T250 35Q231 35 224 44T216 63Q216 80 222 143T229 213L171 171Q115 130 110 127Q106 124 100 124Q87 124 76 134T64 161Q64 166 64 169T67 175T72 181T81 188T94 195T113 204T138 215T170 230T210 250L74 315Q65 324 65 338Q65 353 74 363T98 374Q106 374 116 368T171 328L229 286Z"></path><path id="MJMATHI-3C3" stroke-width="1" d="M184 -11Q116 -11 74 34T31 147Q31 247 104 333T274 430Q275 431 414 431H552Q553 430 555 429T559 427T562 425T565 422T567 420T569 416T570 412T571 407T572 401Q572 357 507 357Q500 357 490 357T476 358H416L421 348Q439 310 439 263Q439 153 359 71T184 -11ZM361 278Q361 358 276 358Q152 358 115 184Q114 180 114 178Q106 141 106 117Q106 67 131 47T188 26Q242 26 287 73Q316 103 334 153T356 233T361 278Z"></path></defs></svg></div><div id="MathJax_Message" style="display: none;"></div>

    <div id="global-prompt-alert" class="hide alert alert-warning">
        <span id="global-prompt-message"></span>
        <a id="close-global-prompt-alert" href="https://www.zybuluo.com/hanbingtao/note/476663">[关闭]</a>
    </div>

    <!-- zybuluo's body -->
    







<!-- mdeditor's body -->






<div id="editor-reader-full" class="editor-reader-full-shown" style="position: static; padding-right: 75px;">
    <div id="reader-full-topInfo" class="reader-full-topInfo-shown">
        <span>
            <code>@hanbingtao</code>
        </span>
        <code><span class="article-updated-date">2017-10-17 22:25</span></code>
        <code><span>字数 </span><span class="article-characters">25454</span></code>
        <code><span>阅读 </span><span class="article-read">255603</span></code>
    </div>
    <div id="wmd-preview" class="wmd-preview wmd-preview-full-reader" data-medium-element="true" style="left: 0px;"><div class="md-section-divider"></div><div class="md-section-divider"></div><h1 data-anchor-id="m46z" id="零基础入门深度学习3-神经网络和反向传播算法">零基础入门深度学习(3) - 神经网络和反向传播算法</h1><p data-anchor-id="f0rd"><code>机器学习</code> <code>深度学习入门</code></p><hr><p data-anchor-id="41in"><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-06627c71f0d8c0dc.jpg" alt=""></p><blockquote data-anchor-id="x7r0" class="white-blockquote">
  <p>无论即将到来的是大数据时代还是人工智能时代，亦或是传统行业使用人工智能在云上处理大数据的时代，作为一个有理想有追求的程序员，不懂深度学习（Deep Learning）这个超热的技术，会不会感觉马上就out了？现在救命稻草来了，《零基础入门深度学习》系列文章旨在讲帮助爱编程的你从零基础达到入门级水平。零基础意味着你不需要太多的数学知识，只要会写程序就行了，没错，这是专门为程序员写的文章。虽然文中会有很多公式你也许看不懂，但同时也会有更多的代码，程序员的你一定能看懂的（我周围是一群狂热的Clean Code程序员，所以我写的代码也不会很差）。</p>
</blockquote><div class="md-section-divider"></div><h2 data-anchor-id="xuc5" id="文章列表">文章列表</h2><p data-anchor-id="uel4"><a href="https://www.zybuluo.com/hanbingtao/note/433855" target="_blank">零基础入门深度学习(1) - 感知器</a> <br>
<a href="https://www.zybuluo.com/hanbingtao/note/448086" target="_blank">零基础入门深度学习(2) - 线性单元和梯度下降</a> <br>
<a href="https://www.zybuluo.com/hanbingtao/note/476663" target="_blank">零基础入门深度学习(3) - 神经网络和反向传播算法</a> <br>
<a href="https://www.zybuluo.com/hanbingtao/note/485480" target="_blank">零基础入门深度学习(4) - 卷积神经网络</a> <br>
<a href="https://zybuluo.com/hanbingtao/note/541458" target="_blank">零基础入门深度学习(5) - 循环神经网络</a> <br>
<a href="https://zybuluo.com/hanbingtao/note/581764" target="_blank">零基础入门深度学习(6) - 长短时记忆网络(LSTM)</a> <br>
<a href="https://zybuluo.com/hanbingtao/note/626300" target="_blank">零基础入门深度学习(7) - 递归神经网络</a></p><div class="md-section-divider"></div><h2 data-anchor-id="l2q6" id="往期回顾">往期回顾</h2><p data-anchor-id="tyid">在上一篇文章中，我们已经掌握了机器学习的基本套路，对模型、目标函数、优化算法这些概念有了一定程度的理解，而且已经会训练单个的感知器或者线性单元了。在这篇文章中，我们将把这些单独的单元按照一定的规则相互连接在一起形成<strong>神经网络</strong>，从而奇迹般的获得了强大的学习能力。我们还将介绍这种网络的训练算法：<strong>反向传播算法</strong>。最后，我们依然用代码实现一个神经网络。如果您能坚持到本文的结尾，将会看到我们用自己实现的神经网络去识别手写数字。现在请做好准备，您即将双手触及到深度学习的大门。</p><div class="md-section-divider"></div><h2 data-anchor-id="lt93" id="神经元">神经元</h2><p data-anchor-id="jhtz">神经元和感知器本质上是一样的，只不过我们说感知器的时候，它的激活函数是<strong>阶跃函数</strong>；而当我们说神经元时，激活函数往往选择为sigmoid函数或tanh函数。如下图所示：</p><p data-anchor-id="hgru"><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-49f06e2e9d3eb29f.gif" alt=""></p><p data-anchor-id="l6hf">计算一个神经元的输出的方法和计算一个感知器的输出是一样的。假设神经元的输入是向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-1-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -740.324787109928 572.5 771.5248564779206" style="width: 1.331ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g></svg></span><script type="math/tex" id="MathJax-Element-1">\vec{x}</script>，权重向量是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-2-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -741.324787109928 716.5 772.5248564779206" style="width: 1.694ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use></g></svg></span><script type="math/tex" id="MathJax-Element-2">\vec{w}</script>(偏置项是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-3-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1170.406943983867 648.6635947032757" style="width: 2.661ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="1013" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-3">w_0</script>)，激活函数是sigmoid函数，则其输出<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-4-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 497.5 688.4001387359851" style="width: 1.21ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use></g></svg></span><script type="math/tex" id="MathJax-Element-4">y</script>：</p><div class="md-section-divider"></div><p data-anchor-id="c4p4"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-5-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -947.7235996840317 12861.987049926569 1331.5056851810566" style="width: 29.879ex; height: 3.145ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use href="#MJMAIN-3D" x="775" y="0"></use><use href="#MJMATHI-73" x="1831" y="0"></use><use href="#MJMATHI-69" x="2301" y="0"></use><use href="#MJMATHI-67" x="2646" y="0"></use><use href="#MJMATHI-6D" x="3127" y="0"></use><use href="#MJMATHI-6F" x="4005" y="0"></use><use href="#MJMATHI-69" x="4491" y="0"></use><use href="#MJMATHI-64" x="4836" y="0"></use><use href="#MJMAIN-28" x="5360" y="0"></use><g transform="translate(5749,0)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="1013" y="635"></use></g><use href="#MJMAIN-22C5" x="7286" y="0"></use><g transform="translate(7787,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><use href="#MJMAIN-29" x="8359" y="0"></use><use href="#MJMAIN-28" x="10749" y="0"></use><g transform="translate(11138,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-31" x="11971" y="0"></use><use href="#MJMAIN-29" x="12472" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-5">
y=sigmoid(\vec{w}^T\centerdot\vec{x})\qquad(式1)
</script><p></p><p data-anchor-id="v3lw">sigmoid函数的定义如下：</p><div class="md-section-divider"></div><p data-anchor-id="di7d"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-6-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1362.7080693679925 9819.301261383025 2150.8591387359847" style="width: 22.863ex; height: 4.96ex; vertical-align: -1.935ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><use href="#MJMATHI-69" x="469" y="0"></use><use href="#MJMATHI-67" x="815" y="0"></use><use href="#MJMATHI-6D" x="1295" y="0"></use><use href="#MJMATHI-6F" x="2174" y="0"></use><use href="#MJMATHI-69" x="2659" y="0"></use><use href="#MJMATHI-64" x="3005" y="0"></use><use href="#MJMAIN-28" x="3528" y="0"></use><use href="#MJMATHI-78" x="3918" y="0"></use><use href="#MJMAIN-29" x="4490" y="0"></use><use href="#MJMAIN-3D" x="5157" y="0"></use><g transform="translate(6334,0)"><rect stroke="none" width="3365" height="60" x="0" y="220"></rect><use href="#MJMAIN-31" x="1432" y="676"></use><g transform="translate(60,-686)"><use href="#MJMAIN-31"></use><use href="#MJMAIN-2B" x="722" y="0"></use><g transform="translate(1723,0)"><use href="#MJMATHI-65"></use><g transform="translate(466,288)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-78" x="778" y="0"></use></g></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-6">
sigmoid(x)=\frac{1}{1+e^{-x}}
</script><p></p><p data-anchor-id="afzy">将其带入前面的式子，得到</p><div class="md-section-divider"></div><p data-anchor-id="1mv4"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-7-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1362.7080693679925 6563.333186782295 2403.451486465283" style="width: 15.242ex; height: 5.565ex; vertical-align: -2.54ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use href="#MJMAIN-3D" x="775" y="0"></use><g transform="translate(1951,0)"><rect stroke="none" width="4491" height="60" x="0" y="220"></rect><use href="#MJMAIN-31" x="1995" y="676"></use><g transform="translate(60,-939)"><use href="#MJMAIN-31"></use><use href="#MJMAIN-2B" x="722" y="0"></use><g transform="translate(1723,0)"><use href="#MJMATHI-65"></use><g transform="translate(466,288)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212"></use><g transform="translate(550,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-77"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-20D7" x="711" y="39"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-54" x="1013" y="680"></use></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-22C5" x="2093" y="0"></use><g transform="translate(1677,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-20D7" x="560" y="38"></use></g></g></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-7">
y=\frac{1}{1+e^{-\vec{w}^T\centerdot\vec{x}}}
</script><p></p><p data-anchor-id="qt2s">sigmoid函数是一个非线性函数，值域是(0,1)。函数图像如下图所示</p><p data-anchor-id="yvzv"><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-e7e64f57dc6b1c64.jpg" alt=""></p><p data-anchor-id="4tgn">sigmoid函数的导数是：</p><div class="md-section-divider"></div><p data-anchor-id="9wwr"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-8-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1669.1955532389602 44270.673387096766 2838.3911064779204" style="width: 102.823ex; height: 6.532ex; vertical-align: -2.782ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(18398,0)"><g transform="translate(263,0)"><g transform="translate(0,764)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">令</text><use href="#MJMATHI-79" x="833" y="0"></use><use href="#MJMAIN-3D" x="1608" y="0"></use><use href="#MJMATHI-73" x="2664" y="0"></use><use href="#MJMATHI-69" x="3134" y="0"></use><use href="#MJMATHI-67" x="3479" y="0"></use><use href="#MJMATHI-6D" x="3960" y="0"></use><use href="#MJMATHI-6F" x="4838" y="0"></use><use href="#MJMATHI-69" x="5324" y="0"></use><use href="#MJMATHI-64" x="5669" y="0"></use><use href="#MJMAIN-28" x="6193" y="0"></use><use href="#MJMATHI-78" x="6582" y="0"></use><use href="#MJMAIN-29" x="7155" y="0"></use></g><g transform="translate(0,-786)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">则</text><g transform="translate(833,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="706" y="583"></use></g><use href="#MJMAIN-3D" x="1905" y="0"></use><use href="#MJMATHI-79" x="2961" y="0"></use><use href="#MJMAIN-28" x="3459" y="0"></use><use href="#MJMAIN-31" x="3848" y="0"></use><use href="#MJMAIN-2212" x="4571" y="0"></use><use href="#MJMATHI-79" x="5572" y="0"></use><use href="#MJMAIN-29" x="6069" y="0"></use></g></g></g><g transform="translate(42191,0)"><g transform="translate(0,764)"><g id="mjx-eqn-1"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-786)"><g id="mjx-eqn-2"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-8">
\begin{align}
&令y=sigmoid(x)\\
&则y'=y(1-y)
\end{align}
</script><p></p><p data-anchor-id="9p0k">可以看到，sigmoid函数的导数非常有趣，它可以用sigmoid函数自身来表示。这样，一旦计算出sigmoid函数的值，计算它的导数的值就非常方便。</p><div class="md-section-divider"></div><h2 data-anchor-id="jhuu" id="神经网络是啥">神经网络是啥</h2><p data-anchor-id="7x2j"><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-92111b104ce0d571.jpeg" alt=""></p><p data-anchor-id="ft98">神经网络其实就是按照<strong>一定规则</strong>连接起来的多个<strong>神经元</strong>。上图展示了一个<strong>全连接(full connected, FC)</strong>神经网络，通过观察上面的图，我们可以发现它的规则包括：</p><ul data-anchor-id="7uzv">
<li>神经元按照<strong>层</strong>来布局。最左边的层叫做<strong>输入层</strong>，负责接收输入数据；最右边的层叫<strong>输出层</strong>，我们可以从这层获取神经网络输出数据。输入层和输出层之间的层叫做<strong>隐藏层</strong>，因为它们对于外部来说是不可见的。</li>
<li>同一层的神经元之间没有连接。</li>
<li>第N层的每个神经元和第N-1层的<strong>所有</strong>神经元相连(这就是full connected的含义)，第N-1层神经元的输出就是第N层神经元的输入。</li>
<li>每个连接都有一个<strong>权值</strong>。</li>
</ul><p data-anchor-id="t4h2">上面这些规则定义了全连接神经网络的结构。事实上还存在很多其它结构的神经网络，比如卷积神经网络(CNN)、循环神经网络(RNN)，他们都具有不同的连接规则。</p><div class="md-section-divider"></div><h2 data-anchor-id="f526" id="计算神经网络的输出">计算神经网络的输出</h2><p data-anchor-id="zmso">神经网络实际上就是一个输入向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-9-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -740.324787109928 572.5 771.5248564779206" style="width: 1.331ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g></svg></span><script type="math/tex" id="MathJax-Element-9">\vec{x}</script>到输出向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-10-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -741.324787109928 544.425 966.5248564779206" style="width: 1.21ex; height: 2.298ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use href="#MJMAIN-20D7" x="543" y="7"></use></g></svg></span><script type="math/tex" id="MathJax-Element-10">\vec{y}</script>的函数，即：</p><div class="md-section-divider"></div><p data-anchor-id="7sfa"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-11-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 6368.53984156128 1040.4001387359851" style="width: 14.758ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use href="#MJMAIN-20D7" x="543" y="7"></use><use href="#MJMAIN-3D" x="822" y="0"></use><g transform="translate(1878,0)"><use href="#MJMATHI-66"></use><g transform="translate(490,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="600" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1067" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-77" x="1428" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F" x="2145" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-72" x="2630" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="3082" y="0"></use></g></g><use href="#MJMAIN-28" x="5017" y="0"></use><g transform="translate(5406,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><use href="#MJMAIN-29" x="5979" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-11">
\vec{y} = f_{network}(\vec{x})
</script><p></p><p data-anchor-id="4eh9">根据输入计算神经网络的输出，需要首先将输入向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-12-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -740.324787109928 572.5 771.5248564779206" style="width: 1.331ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g></svg></span><script type="math/tex" id="MathJax-Element-12">\vec{x}</script>的每个元素<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-13-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 916.8053928999522 639.8854201102238" style="width: 2.177ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="809" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-13">x_i</script>的值赋给神经网络的输入层的对应神经元，然后根据<strong>式1</strong>依次向前计算每一层的每个神经元的值，直到最后一层输出层的所有神经元的值计算完毕。最后，将输出层每个神经元的值串在一起就得到了输出向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-14-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -741.324787109928 544.425 966.5248564779206" style="width: 1.21ex; height: 2.298ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use href="#MJMAIN-20D7" x="543" y="7"></use></g></svg></span><script type="math/tex" id="MathJax-Element-14">\vec{y}</script>。</p><p data-anchor-id="xqw3">接下来举一个例子来说明这个过程，我们先给神经网络的每个单元写上编号。</p><p data-anchor-id="zp36"><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-bfbb364740f898d1.png" alt="" title=""></p><p data-anchor-id="ykzb">如上图，输入层有三个节点，我们将其依次编号为1、2、3；隐藏层的4个节点，编号依次为4、5、6、7；最后输出层的两个节点编号为8、9。因为我们这个神经网络是<strong>全连接</strong>网络，所以可以看到每个节点都和<strong>上一层的所有节点</strong>有连接。比如，我们可以看到隐藏层的节点4，它和输入层的三个节点1、2、3之间都有连接，其连接上的权重分别为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-15-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 5463.2749972365355 678.4001387359851" style="width: 12.702ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><use href="#MJMAIN-2C" x="1524" y="0"></use><g transform="translate(1969,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="3493" y="0"></use><g transform="translate(3938,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-15">w_{41},w_{42},w_{43}</script>。那么，我们怎样计算节点4的输出值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-16-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -461.20006936799257 983.406943983867 631.1072455171717" style="width: 2.298ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="748" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-16">a_4</script>呢？</p><p data-anchor-id="frle">为了计算节点4的输出值，我们必须先得到其所有上游节点（也就是节点1、2、3）的输出值。节点1、2、3是<strong>输入层</strong>的节点，所以，他们的输出值就是输入向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-17-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -740.324787109928 572.5 771.5248564779206" style="width: 1.331ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g></svg></span><script type="math/tex" id="MathJax-Element-17">\vec{x}</script>本身。按照上图画出的对应关系，可以看到节点1、2、3的输出值分别是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-18-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 3969.5541652849342 677.4001387359851" style="width: 9.194ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use><use href="#MJMAIN-2C" x="1026" y="0"></use><g transform="translate(1471,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-213"></use></g><use href="#MJMAIN-2C" x="2497" y="0"></use><g transform="translate(2943,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="809" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-18">x_1,x_2,x_3</script>。我们要求<strong>输入向量的维度和输入层神经元个数相同</strong>，而输入向量的某个元素对应到哪个输入节点是可以自由决定的，你偏非要把<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-19-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 1026.406943983867 632.1072455171717" style="width: 2.419ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-19">x_1</script>赋值给节点2也是完全没有问题的，但这样除了把自己弄晕之外，并没有什么价值。</p><p data-anchor-id="o1kt">一旦我们有了节点1、2、3的输出值，我们就可以根据<strong>式1</strong>计算节点4的输出值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-20-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -461.20006936799257 983.406943983867 631.1072455171717" style="width: 2.298ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="748" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-20">a_4</script>：</p><div class="md-section-divider"></div><p data-anchor-id="cv1g"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-21-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1509.961834526012 44270.673387096766 2519.923669052024" style="width: 102.823ex; height: 5.806ex; vertical-align: -2.419ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(12599,0)"><g transform="translate(-15,0)"><g transform="translate(0,562)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="748" y="-213"></use></g></g><g transform="translate(1247,0)"><g transform="translate(0,562)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-73" x="1056" y="0"></use><use href="#MJMATHI-69" x="1525" y="0"></use><use href="#MJMATHI-67" x="1871" y="0"></use><use href="#MJMATHI-6D" x="2351" y="0"></use><use href="#MJMATHI-6F" x="3230" y="0"></use><use href="#MJMATHI-69" x="3715" y="0"></use><use href="#MJMATHI-64" x="4061" y="0"></use><use href="#MJMAIN-28" x="4584" y="0"></use><g transform="translate(4974,0)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="1013" y="635"></use></g><use href="#MJMAIN-22C5" x="6511" y="0"></use><g transform="translate(7011,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><use href="#MJMAIN-29" x="7584" y="0"></use></g><g transform="translate(0,-740)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-73" x="1056" y="0"></use><use href="#MJMATHI-69" x="1525" y="0"></use><use href="#MJMATHI-67" x="1871" y="0"></use><use href="#MJMATHI-6D" x="2351" y="0"></use><use href="#MJMATHI-6F" x="3230" y="0"></use><use href="#MJMATHI-69" x="3715" y="0"></use><use href="#MJMATHI-64" x="4061" y="0"></use><use href="#MJMAIN-28" x="4584" y="0"></use><g transform="translate(4974,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><g transform="translate(6498,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="7747" y="0"></use><g transform="translate(8747,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(10272,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="11520" y="0"></use><g transform="translate(12521,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><g transform="translate(14045,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="15294" y="0"></use><g transform="translate(16295,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g><use href="#MJMAIN-29" x="17769" y="0"></use></g></g></g><g transform="translate(42191,0)"><g transform="translate(0,562)"><g id="mjx-eqn-3"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-740)"><g id="mjx-eqn-4"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-21">
\begin{align}
a_4&=sigmoid(\vec{w}^T\centerdot\vec{x})\\
&=sigmoid(w_{41}x_1+w_{42}x_2+w_{43}x_3+w_{4b})
\end{align}
</script><p></p><p data-anchor-id="11vf">上式的<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-22-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1474.1093065034893 640.8854201102238" style="width: 3.387ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-22">w_{4b}</script>是节点4的<strong>偏置项</strong>，图中没有画出来。而<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-23-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 5463.2749972365355 678.4001387359851" style="width: 12.702ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><use href="#MJMAIN-2C" x="1524" y="0"></use><g transform="translate(1969,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="3493" y="0"></use><g transform="translate(3938,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-23">w_{41},w_{42},w_{43}</script>分别为节点1、2、3到节点4连接的权重，在给权重<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-24-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-24">w_{ji}</script>编号时，我们把目标节点的编号<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-25-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-11.5 -681.2000693679926 424 905.4001387359851" style="width: 0.968ex; height: 2.056ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6A"></use></g></svg></span><script type="math/tex" id="MathJax-Element-25">j</script>放在前面，把源节点的编号<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-26-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-26">i</script>放在后面。</p><p data-anchor-id="cq60">同样，我们可以继续计算出节点5、6、7的输出值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-27-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -461.20006936799257 3840.5541652849342 676.4001387359851" style="width: 8.952ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-35" x="748" y="-213"></use><use href="#MJMAIN-2C" x="983" y="0"></use><g transform="translate(1428,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-36" x="748" y="-213"></use></g><use href="#MJMAIN-2C" x="2411" y="0"></use><g transform="translate(2857,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-37" x="748" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-27">a_5,a_6,a_7</script>。这样，隐藏层的4个节点的输出值就计算完成了，我们就可以接着计算输出层的节点8的输出值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-28-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 944.406943983867 688.4001387359851" style="width: 2.177ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-28">y_1</script>：</p><div class="md-section-divider"></div><p data-anchor-id="nvvo"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-198-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1509.961834526012 44270.673387096766 2519.923669052024" style="width: 102.823ex; height: 5.806ex; vertical-align: -2.419ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(10818,0)"><g transform="translate(-15,0)"><g transform="translate(0,562)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="693" y="-213"></use></g></g><g transform="translate(1208,0)"><g transform="translate(0,562)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-73" x="1056" y="0"></use><use href="#MJMATHI-69" x="1525" y="0"></use><use href="#MJMATHI-67" x="1871" y="0"></use><use href="#MJMATHI-6D" x="2351" y="0"></use><use href="#MJMATHI-6F" x="3230" y="0"></use><use href="#MJMATHI-69" x="3715" y="0"></use><use href="#MJMATHI-64" x="4061" y="0"></use><use href="#MJMAIN-28" x="4584" y="0"></use><g transform="translate(4974,0)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="1013" y="635"></use></g><use href="#MJMAIN-22C5" x="6511" y="0"></use><g transform="translate(7011,0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use></g><use href="#MJMAIN-29" x="7541" y="0"></use></g><g transform="translate(0,-740)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-73" x="1056" y="0"></use><use href="#MJMATHI-69" x="1525" y="0"></use><use href="#MJMATHI-67" x="1871" y="0"></use><use href="#MJMATHI-6D" x="2351" y="0"></use><use href="#MJMATHI-6F" x="3230" y="0"></use><use href="#MJMATHI-69" x="3715" y="0"></use><use href="#MJMATHI-64" x="4061" y="0"></use><use href="#MJMAIN-28" x="4584" y="0"></use><g transform="translate(4974,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-38"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="500" y="0"></use></g></g><g transform="translate(6498,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="748" y="-213"></use></g><use href="#MJMAIN-2B" x="7704" y="0"></use><g transform="translate(8704,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-38"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-35" x="500" y="0"></use></g></g><g transform="translate(10229,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-35" x="748" y="-213"></use></g><use href="#MJMAIN-2B" x="11434" y="0"></use><g transform="translate(12435,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-38"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-36" x="500" y="0"></use></g></g><g transform="translate(13959,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-36" x="748" y="-213"></use></g><use href="#MJMAIN-2B" x="15165" y="0"></use><g transform="translate(16166,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-38"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-37" x="500" y="0"></use></g></g><g transform="translate(17690,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-37" x="748" y="-213"></use></g><use href="#MJMAIN-2B" x="18896" y="0"></use><g transform="translate(19896,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-38"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g><use href="#MJMAIN-29" x="21371" y="0"></use></g></g></g><g transform="translate(42191,0)"><g transform="translate(0,562)"><g id="mjx-eqn-5"><use href="#MJMAIN-28"></use><use href="#MJMAIN-35" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-740)"><g id="mjx-eqn-6"><use href="#MJMAIN-28"></use><use href="#MJMAIN-36" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-198">
\begin{align}
y_1&=sigmoid(\vec{w}^T\centerdot\vec{a})\\
&=sigmoid(w_{84}a_4+w_{85}a_5+w_{86}a_6+w_{87}a_7+w_{8b})
\end{align}
</script><p></p><p data-anchor-id="isur">同理，我们还可以计算出<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-30-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 944.406943983867 688.4001387359851" style="width: 2.177ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-30">y_2</script>的值。这样输出层所有节点的输出值计算完毕，我们就得到了在输入向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-31-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1969.2000693679925 4587.562447513428 3438.400138735985" style="width: 10.645ex; height: 7.984ex; vertical-align: -3.508ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use><use href="#MJMAIN-3D" x="850" y="0"></use><g transform="translate(1906,0)"><g transform="translate(0,1950)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><use href="#MJSZ4-23A3" x="0" y="-2755"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,1150)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g><g transform="translate(0,-50)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-213"></use></g><g transform="translate(0,-1250)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="809" y="-213"></use></g></g></g><g transform="translate(2013,1950)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><use href="#MJSZ4-23A6" x="0" y="-2755"></use></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-31">\vec{x}=\begin{bmatrix}x_1\\x_2\\x_3\end{bmatrix}</script>时，神经网络的输出向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-32-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1469.7000693679925 4199.487447513428 2439.400138735985" style="width: 9.798ex; height: 5.685ex; vertical-align: -2.298ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use href="#MJMAIN-20D7" x="543" y="7"></use><use href="#MJMAIN-3D" x="822" y="0"></use><g transform="translate(1878,0)"><use href="#MJSZ3-5B"></use><g transform="translate(695,0)"><g transform="translate(-15,0)"><g transform="translate(0,553)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="693" y="-213"></use></g><g transform="translate(0,-647)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="693" y="-213"></use></g></g></g><use href="#MJSZ3-5D" x="1792" y="-1"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-32">\vec{y}=\begin{bmatrix}y_1\\y_2\end{bmatrix}</script>。这里我们也看到，<strong>输出向量的维度和输出层神经元个数相同</strong>。</p><div class="md-section-divider"></div><h3 data-anchor-id="m8jq" id="神经网络的矩阵表示">神经网络的矩阵表示</h3><p data-anchor-id="3sfs">神经网络的计算如果用矩阵来表示会很方便（当然逼格也更高），我们先来看看隐藏层的矩阵表示。</p><p data-anchor-id="kz35">首先我们把隐藏层4个节点的计算依次排列出来：</p><div class="md-section-divider"></div><p data-anchor-id="74ly"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true"><span class="MathJax_SVG" id="MathJax-Element-33-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 44270.673387096766 6287.062138735985" style="width: 102.823ex; height: 14.637ex; vertical-align: -12.944ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(12425,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="748" y="-213"></use><use href="#MJMAIN-3D" x="1261" y="0"></use><use href="#MJMATHI-73" x="2317" y="0"></use><use href="#MJMATHI-69" x="2786" y="0"></use><use href="#MJMATHI-67" x="3132" y="0"></use><use href="#MJMATHI-6D" x="3612" y="0"></use><use href="#MJMATHI-6F" x="4491" y="0"></use><use href="#MJMATHI-69" x="4976" y="0"></use><use href="#MJMATHI-64" x="5322" y="0"></use><use href="#MJMAIN-28" x="5845" y="0"></use><g transform="translate(6235,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><g transform="translate(7759,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="9008" y="0"></use><g transform="translate(10009,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(11533,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="12782" y="0"></use><g transform="translate(13782,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><g transform="translate(15307,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="16555" y="0"></use><g transform="translate(17556,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g><use href="#MJMAIN-29" x="19030" y="0"></use></g><g transform="translate(12425,-1433)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-35" x="748" y="-213"></use><use href="#MJMAIN-3D" x="1261" y="0"></use><use href="#MJMATHI-73" x="2317" y="0"></use><use href="#MJMATHI-69" x="2786" y="0"></use><use href="#MJMATHI-67" x="3132" y="0"></use><use href="#MJMATHI-6D" x="3612" y="0"></use><use href="#MJMATHI-6F" x="4491" y="0"></use><use href="#MJMATHI-69" x="4976" y="0"></use><use href="#MJMATHI-64" x="5322" y="0"></use><use href="#MJMAIN-28" x="5845" y="0"></use><g transform="translate(6235,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><g transform="translate(7759,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="9008" y="0"></use><g transform="translate(10009,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(11533,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="12782" y="0"></use><g transform="translate(13782,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><g transform="translate(15307,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="16555" y="0"></use><g transform="translate(17556,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g><use href="#MJMAIN-29" x="19030" y="0"></use></g><g transform="translate(12425,-2866)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-36" x="748" y="-213"></use><use href="#MJMAIN-3D" x="1261" y="0"></use><use href="#MJMATHI-73" x="2317" y="0"></use><use href="#MJMATHI-69" x="2786" y="0"></use><use href="#MJMATHI-67" x="3132" y="0"></use><use href="#MJMATHI-6D" x="3612" y="0"></use><use href="#MJMATHI-6F" x="4491" y="0"></use><use href="#MJMATHI-69" x="4976" y="0"></use><use href="#MJMATHI-64" x="5322" y="0"></use><use href="#MJMAIN-28" x="5845" y="0"></use><g transform="translate(6235,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><g transform="translate(7759,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="9008" y="0"></use><g transform="translate(10009,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(11533,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="12782" y="0"></use><g transform="translate(13782,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><g transform="translate(15307,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="16555" y="0"></use><g transform="translate(17556,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g><use href="#MJMAIN-29" x="19030" y="0"></use></g><g transform="translate(12425,-4298)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-37" x="748" y="-213"></use><use href="#MJMAIN-3D" x="1261" y="0"></use><use href="#MJMATHI-73" x="2317" y="0"></use><use href="#MJMATHI-69" x="2786" y="0"></use><use href="#MJMATHI-67" x="3132" y="0"></use><use href="#MJMATHI-6D" x="3612" y="0"></use><use href="#MJMATHI-6F" x="4491" y="0"></use><use href="#MJMATHI-69" x="4976" y="0"></use><use href="#MJMATHI-64" x="5322" y="0"></use><use href="#MJMAIN-28" x="5845" y="0"></use><g transform="translate(6235,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><g transform="translate(7759,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="9008" y="0"></use><g transform="translate(10009,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><g transform="translate(11533,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="12782" y="0"></use><g transform="translate(13782,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><g transform="translate(15307,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="809" y="-213"></use></g><use href="#MJMAIN-2B" x="16555" y="0"></use><g transform="translate(17556,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g><use href="#MJMAIN-29" x="19030" y="0"></use></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-33">
a_4=sigmoid(w_{41}x_1+w_{42}x_2+w_{43}x_3+w_{4b})\\
a_5=sigmoid(w_{51}x_1+w_{52}x_2+w_{53}x_3+w_{5b})\\
a_6=sigmoid(w_{61}x_1+w_{62}x_2+w_{63}x_3+w_{6b})\\
a_7=sigmoid(w_{71}x_1+w_{72}x_2+w_{73}x_3+w_{7b})\\
</script><p></p><p data-anchor-id="h72c">接着，定义网络的输入向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-34-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -740.324787109928 572.5 771.5248564779206" style="width: 1.331ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g></svg></span><script type="math/tex" id="MathJax-Element-34">\vec{x}</script>和隐藏层每个节点的权重向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-35-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1065.324787109928 1108.181547239451 1379.4817466211628" style="width: 2.54ex; height: 3.145ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="1013" y="-213"></use><use href="#MJMAIN-2192" x="53" y="534"></use></g></svg></span><script type="math/tex" id="MathJax-Element-35">\vec{w_j}</script>。令</p><div class="md-section-divider"></div><p data-anchor-id="vd0n"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-36-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -5840.831797351639 44270.673387096766 11181.663594703277" style="width: 102.823ex; height: 26.008ex; vertical-align: -12.46ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(17087,0)"><g transform="translate(-15,0)"><g transform="translate(597,3255)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><g transform="translate(0,137)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="1013" y="-213"></use></g><g transform="translate(0,-1165)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-35" x="1013" y="-213"></use></g><g transform="translate(0,-2467)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-36" x="1013" y="-213"></use></g><g transform="translate(0,-3769)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-37" x="1013" y="-213"></use></g><use href="#MJMATHI-66" x="619" y="-5071"></use></g><g transform="translate(1434,0)"><g transform="translate(0,3255)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><g transform="translate(0,2566)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-2880.390503225623) scale(1,1.8729527416676996)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-3989"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,1766)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g><g transform="translate(0,566)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-213"></use></g><g transform="translate(0,-634)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="809" y="-213"></use></g><use href="#MJMAIN-31" x="262" y="-1867"></use></g></g><g transform="translate(2013,2566)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-2880.390503225623) scale(1,1.8729527416676996)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-3989"></use></g></g></g><g transform="translate(0,137)"><use href="#MJMAIN-3D"></use><use href="#MJMAIN-5B" x="1056" y="0"></use><g transform="translate(1334,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="2859" y="0"></use><g transform="translate(3304,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="4828" y="0"></use><g transform="translate(5273,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="6798" y="0"></use><g transform="translate(7243,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g><use href="#MJMAIN-5D" x="8717" y="0"></use></g><g transform="translate(0,-1165)"><use href="#MJMAIN-3D"></use><use href="#MJMAIN-5B" x="1056" y="0"></use><g transform="translate(1334,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="2859" y="0"></use><g transform="translate(3304,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="4828" y="0"></use><g transform="translate(5273,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="6798" y="0"></use><g transform="translate(7243,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g><use href="#MJMAIN-5D" x="8717" y="0"></use></g><g transform="translate(0,-2467)"><use href="#MJMAIN-3D"></use><use href="#MJMAIN-5B" x="1056" y="0"></use><g transform="translate(1334,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="2859" y="0"></use><g transform="translate(3304,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="4828" y="0"></use><g transform="translate(5273,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="6798" y="0"></use><g transform="translate(7243,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g><use href="#MJMAIN-5D" x="8717" y="0"></use></g><g transform="translate(0,-3769)"><use href="#MJMAIN-3D"></use><use href="#MJMAIN-5B" x="1056" y="0"></use><g transform="translate(1334,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="2859" y="0"></use><g transform="translate(3304,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="4828" y="0"></use><g transform="translate(5273,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="6798" y="0"></use><g transform="translate(7243,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g><use href="#MJMAIN-5D" x="8717" y="0"></use></g><g transform="translate(0,-5071)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-73" x="1056" y="0"></use><use href="#MJMATHI-69" x="1525" y="0"></use><use href="#MJMATHI-67" x="1871" y="0"></use><use href="#MJMATHI-6D" x="2351" y="0"></use><use href="#MJMATHI-6F" x="3230" y="0"></use><use href="#MJMATHI-69" x="3715" y="0"></use><use href="#MJMATHI-64" x="4061" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(500,3255)"><g id="mjx-eqn-7"><use href="#MJMAIN-28"></use><use href="#MJMAIN-37" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(500,137)"><g id="mjx-eqn-8"><use href="#MJMAIN-28"></use><use href="#MJMAIN-38" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(500,-1165)"><g id="mjx-eqn-9"><use href="#MJMAIN-28"></use><use href="#MJMAIN-39" x="389" y="0"></use><use href="#MJMAIN-29" x="890" y="0"></use></g></g><g transform="translate(0,-2467)"><g id="mjx-eqn-10"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-3769)"><g id="mjx-eqn-11"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-5071)"><g id="mjx-eqn-12"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-36">
\begin{align}
\vec{x}&=\begin{bmatrix}x_1\\x_2\\x_3\\1\end{bmatrix}\\
\vec{w}_4&=[w_{41},w_{42},w_{43},w_{4b}]\\
\vec{w}_5&=[w_{51},w_{52},w_{53},w_{5b}]\\
\vec{w}_6&=[w_{61},w_{62},w_{63},w_{6b}]\\
\vec{w}_7&=[w_{71},w_{72},w_{73},w_{7b}]\\
f&=sigmoid
\end{align}
</script><p></p><p data-anchor-id="0a6h">代入到前面的一组式子，得到：</p><div class="md-section-divider"></div><p data-anchor-id="6a6w"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-37-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3313.4495048518634 44270.673387096766 6126.899009703727" style="width: 102.823ex; height: 14.274ex; vertical-align: -6.653ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(19253,0)"><g transform="translate(-15,0)"><g transform="translate(0,2248)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="748" y="-213"></use></g><g transform="translate(0,651)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-35" x="748" y="-213"></use></g><g transform="translate(0,-947)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-36" x="748" y="-213"></use></g><g transform="translate(0,-2544)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-37" x="748" y="-213"></use></g></g><g transform="translate(1247,0)"><g transform="translate(0,2248)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><g transform="translate(1996,0)"><use href="#MJMATHI-77"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="1013" y="-213"></use><use href="#MJMAIN-2192" x="84" y="534"></use></g><use href="#MJMAIN-22C5" x="3388" y="0"></use><g transform="translate(3889,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><use href="#MJMAIN-29" x="4462" y="0"></use></g><g transform="translate(0,651)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><g transform="translate(1996,0)"><use href="#MJMATHI-77"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-35" x="1013" y="-213"></use><use href="#MJMAIN-2192" x="84" y="534"></use></g><use href="#MJMAIN-22C5" x="3388" y="0"></use><g transform="translate(3889,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><use href="#MJMAIN-29" x="4462" y="0"></use></g><g transform="translate(0,-947)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><g transform="translate(1996,0)"><use href="#MJMATHI-77"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-36" x="1013" y="-213"></use><use href="#MJMAIN-2192" x="84" y="534"></use></g><use href="#MJMAIN-22C5" x="3388" y="0"></use><g transform="translate(3889,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><use href="#MJMAIN-29" x="4462" y="0"></use></g><g transform="translate(0,-2544)"><use href="#MJMAIN-3D"></use><use href="#MJMATHI-66" x="1056" y="0"></use><use href="#MJMAIN-28" x="1606" y="0"></use><g transform="translate(1996,0)"><use href="#MJMATHI-77"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-37" x="1013" y="-213"></use><use href="#MJMAIN-2192" x="84" y="534"></use></g><use href="#MJMAIN-22C5" x="3388" y="0"></use><g transform="translate(3889,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><use href="#MJMAIN-29" x="4462" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,2248)"><g id="mjx-eqn-13"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,651)"><g id="mjx-eqn-14"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-947)"><g id="mjx-eqn-15"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-2544)"><g id="mjx-eqn-16"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-37">
\begin{align}
a_4&=f(\vec{w_4}\centerdot\vec{x})\\
a_5&=f(\vec{w_5}\centerdot\vec{x})\\
a_6&=f(\vec{w_6}\centerdot\vec{x})\\
a_7&=f(\vec{w_7}\centerdot\vec{x})
\end{align}
</script><p></p><p data-anchor-id="ai10">现在，我们把上述计算<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-38-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -461.20006936799257 5269.127775935468 676.4001387359851" style="width: 12.218ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="748" y="-213"></use><use href="#MJMAIN-2C" x="983" y="0"></use><g transform="translate(1428,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-35" x="748" y="-213"></use></g><use href="#MJMAIN-2C" x="2411" y="0"></use><g transform="translate(2857,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-36" x="748" y="-213"></use></g><use href="#MJMAIN-2C" x="3840" y="0"></use><g transform="translate(4285,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-37" x="748" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-38">a_4,a_5,a_6,a_7</script>的四个式子写到一个矩阵里面，每个式子作为矩阵的一行，就可以利用矩阵来表示它们的计算了。令</p><div class="md-section-divider"></div><p data-anchor-id="q4mz"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-39-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3971.2000693679925 34325.73404176774 7442.400138735985" style="width: 79.718ex; height: 17.298ex; vertical-align: -8.105ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use href="#MJMAIN-3D" x="807" y="0"></use><g transform="translate(1863,0)"><g transform="translate(0,2550)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-2847.182119205298) scale(1,1.8178807947019868)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-3955"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,1750)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="748" y="-213"></use></g><g transform="translate(0,550)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-35" x="748" y="-213"></use></g><g transform="translate(0,-650)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-36" x="748" y="-213"></use></g><g transform="translate(0,-1850)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-37" x="748" y="-213"></use></g></g></g><g transform="translate(1970,2550)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-2847.182119205298) scale(1,1.8178807947019868)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-3955"></use></g></g><use href="#MJMAIN-2C" x="4501" y="0"></use><use href="#MJMATHI-57" x="6946" y="0"></use><use href="#MJMAIN-3D" x="8273" y="0"></use><g transform="translate(9329,0)"><g transform="translate(0,2674)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-3096.377032584055) scale(1,2.2311393575191625)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-4205"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,1874)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="1013" y="-213"></use></g><g transform="translate(0,601)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-35" x="1013" y="-213"></use></g><g transform="translate(0,-687)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-36" x="1013" y="-213"></use></g><g transform="translate(0,-1975)"><use href="#MJMATHI-77"></use><use href="#MJMAIN-20D7" x="676" y="7"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-37" x="1013" y="-213"></use></g></g></g><g transform="translate(2157,2674)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-3096.377032584055) scale(1,2.2311393575191625)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-4205"></use></g></g><use href="#MJMAIN-3D" x="12432" y="0"></use><g transform="translate(13488,0)"><g transform="translate(0,2550)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-2847.182119205298) scale(1,1.8178807947019868)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-3955"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,1750)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><use href="#MJMAIN-2C" x="1524" y="0"></use><g transform="translate(1969,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="3493" y="0"></use><g transform="translate(3938,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="5463" y="0"></use><g transform="translate(5908,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g></g><g transform="translate(0,550)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><use href="#MJMAIN-2C" x="1524" y="0"></use><g transform="translate(1969,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="3493" y="0"></use><g transform="translate(3938,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="5463" y="0"></use><g transform="translate(5908,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-35"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g></g><g transform="translate(0,-650)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><use href="#MJMAIN-2C" x="1524" y="0"></use><g transform="translate(1969,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="3493" y="0"></use><g transform="translate(3938,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="5463" y="0"></use><g transform="translate(5908,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-36"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g></g><g transform="translate(0,-1850)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><use href="#MJMAIN-2C" x="1524" y="0"></use><g transform="translate(1969,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="3493" y="0"></use><g transform="translate(3938,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="500" y="0"></use></g></g><use href="#MJMAIN-2C" x="5463" y="0"></use><g transform="translate(5908,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-37"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g></g></g></g><g transform="translate(8369,2550)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-2847.182119205298) scale(1,1.8178807947019868)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-3955"></use></g></g><use href="#MJMAIN-2C" x="22525" y="0"></use><use href="#MJMATHI-66" x="24970" y="0"></use><use href="#MJMAIN-28" x="25521" y="0"></use><g transform="translate(25910,0)"><g transform="translate(0,3750)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-5243.2086092715235) scale(1,5.791390728476821)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-6355"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,2950)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g><g transform="translate(0,1750)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-213"></use></g><g transform="translate(0,550)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="809" y="-213"></use></g><use href="#MJMAIN-2E" x="373" y="-650"></use><use href="#MJMAIN-2E" x="373" y="-1850"></use><use href="#MJMAIN-2E" x="373" y="-3050"></use></g></g><g transform="translate(2013,3750)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-5243.2086092715235) scale(1,5.791390728476821)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-6355"></use></g></g><use href="#MJMAIN-29" x="28591" y="0"></use><use href="#MJMAIN-3D" x="29258" y="0"></use><g transform="translate(30315,0)"><g transform="translate(0,3952)"><use href="#MJSZ4-23A1" x="0" y="-1155"></use><g transform="translate(0,-5646.539735099338) scale(1,6.460264900662252)"><use href="#MJSZ4-23A2"></use></g><use href="#MJSZ4-23A3" x="0" y="-6759"></use></g><g transform="translate(834,0)"><g transform="translate(-15,0)"><g transform="translate(0,3152)"><use href="#MJMATHI-66"></use><use href="#MJMAIN-28" x="550" y="0"></use><g transform="translate(940,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g><use href="#MJMAIN-29" x="1966" y="0"></use></g><g transform="translate(0,1750)"><use href="#MJMATHI-66"></use><use href="#MJMAIN-28" x="550" y="0"></use><g transform="translate(940,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="809" y="-213"></use></g><use href="#MJMAIN-29" x="1966" y="0"></use></g><g transform="translate(0,348)"><use href="#MJMATHI-66"></use><use href="#MJMAIN-28" x="550" y="0"></use><g transform="translate(940,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="809" y="-213"></use></g><use href="#MJMAIN-29" x="1966" y="0"></use></g><use href="#MJMAIN-2E" x="1038" y="-852"></use><use href="#MJMAIN-2E" x="1038" y="-2052"></use><use href="#MJMAIN-2E" x="1038" y="-3252"></use></g></g><g transform="translate(3343,3952)"><use href="#MJSZ4-23A4" x="0" y="-1155"></use><g transform="translate(0,-5646.539735099338) scale(1,6.460264900662252)"><use href="#MJSZ4-23A5"></use></g><use href="#MJSZ4-23A6" x="0" y="-6759"></use></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-39">
\vec{a}=
\begin{bmatrix}
a_4 \\
a_5 \\
a_6 \\
a_7 \\
\end{bmatrix},\qquad W=
\begin{bmatrix}
\vec{w}_4 \\
\vec{w}_5 \\
\vec{w}_6 \\
\vec{w}_7 \\
\end{bmatrix}=
\begin{bmatrix}
w_{41},w_{42},w_{43},w_{4b} \\
w_{51},w_{52},w_{53},w_{5b} \\
w_{61},w_{62},w_{63},w_{6b} \\
w_{71},w_{72},w_{73},w_{7b} \\
\end{bmatrix}
,\qquad f(
\begin{bmatrix}
x_1\\
x_2\\
x_3\\
.\\
.\\
.\\
\end{bmatrix})=
\begin{bmatrix}
f(x_1)\\
f(x_2)\\
f(x_3)\\
.\\
.\\
.\\
\end{bmatrix}
</script><p></p><p data-anchor-id="kkyi">带入前面的一组式子，得到</p><div class="md-section-divider"></div><p data-anchor-id="dpnz"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-40-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -904.613537109928 9649.830322580645 1288.3956226069529" style="width: 22.379ex; height: 3.024ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use href="#MJMAIN-3D" x="807" y="0"></use><use href="#MJMATHI-66" x="1863" y="0"></use><use href="#MJMAIN-28" x="2414" y="0"></use><use href="#MJMATHI-57" x="2803" y="0"></use><use href="#MJMAIN-22C5" x="4074" y="0"></use><g transform="translate(4575,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><use href="#MJMAIN-29" x="5147" y="0"></use><use href="#MJMAIN-28" x="7537" y="0"></use><g transform="translate(7926,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-32" x="8759" y="0"></use><use href="#MJMAIN-29" x="9260" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-40">
\vec{a}=f(W\centerdot\vec{x})\qquad (式2)
</script><p></p><p data-anchor-id="uchd">在<strong>式2</strong>中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-41-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -725.2000693679926 550.5 950.4001387359851" style="width: 1.331ex; height: 2.177ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-66"></use></g></svg></span><script type="math/tex" id="MathJax-Element-41">f</script>是激活函数，在本例中是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-42-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 3528.5 939.4001387359851" style="width: 8.226ex; height: 2.177ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><use href="#MJMATHI-69" x="469" y="0"></use><use href="#MJMATHI-67" x="815" y="0"></use><use href="#MJMATHI-6D" x="1295" y="0"></use><use href="#MJMATHI-6F" x="2174" y="0"></use><use href="#MJMATHI-69" x="2659" y="0"></use><use href="#MJMATHI-64" x="3005" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-42">sigmoid</script>函数；<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-43-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1048.5 745.4001387359851" style="width: 2.419ex; height: 1.694ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use></g></svg></span><script type="math/tex" id="MathJax-Element-43">W</script>是某一层的权重矩阵；<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-44-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -740.324787109928 572.5 771.5248564779206" style="width: 1.331ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g></svg></span><script type="math/tex" id="MathJax-Element-44">\vec{x}</script>是某层的输入向量；<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-45-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -739.324787109928 529.5 769.5248564779206" style="width: 1.21ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use></g></svg></span><script type="math/tex" id="MathJax-Element-45">\vec{a}</script>是某层的输出向量。<strong>式2</strong>说明神经网络的每一层的作用实际上就是先将输入向量<strong>左乘</strong>一个数组进行线性变换，得到一个新的向量，然后再对这个向量<strong>逐元素</strong>应用一个激活函数。</p><p data-anchor-id="msvf">每一层的算法都是一样的。比如，对于包含一个输入层，一个输出层和三个隐藏层的神经网络，我们假设其权重矩阵分别为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-46-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 6929.127775935468 918.4001387359851" style="width: 16.089ex; height: 2.177ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1335" y="-213"></use><use href="#MJMAIN-2C" x="1398" y="0"></use><g transform="translate(1843,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1335" y="-213"></use></g><use href="#MJMAIN-2C" x="3241" y="0"></use><g transform="translate(3687,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="1335" y="-213"></use></g><use href="#MJMAIN-2C" x="5085" y="0"></use><g transform="translate(5530,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="1335" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-46">W_1,W_2,W_3,W_4</script>，每个隐藏层的输出分别是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-47-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -739.324787109928 3840.5541652849342 954.5248564779206" style="width: 8.952ex; height: 2.177ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="748" y="-213"></use><use href="#MJMAIN-2C" x="983" y="0"></use><g transform="translate(1428,0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="748" y="-213"></use></g><use href="#MJMAIN-2C" x="2411" y="0"></use><g transform="translate(2857,0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="748" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-47">\vec{a}_1,\vec{a}_2,\vec{a}_3</script>，神经网络的输入为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-48-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -740.324787109928 572.5 771.5248564779206" style="width: 1.331ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g></svg></span><script type="math/tex" id="MathJax-Element-48">\vec{x}</script>，神经网络的输入为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-49-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -741.324787109928 544.425 966.5248564779206" style="width: 1.21ex; height: 2.298ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use href="#MJMAIN-20D7" x="543" y="7"></use></g></svg></span><script type="math/tex" id="MathJax-Element-49">\vec{y}</script>，如下图所示：</p><p data-anchor-id="bmpe"><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-c1388dc8fdcce427.png" alt="" title=""></p><p data-anchor-id="lnrm">则每一层的输出向量的计算可以表示为：</p><div class="md-section-divider"></div><p data-anchor-id="t61o"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-50-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -2747.7000693679925 44270.673387096766 4995.400138735985" style="width: 102.823ex; height: 11.613ex; vertical-align: -5.323ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(18794,0)"><g transform="translate(263,0)"><g transform="translate(0,1928)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="748" y="-213"></use><use href="#MJMAIN-3D" x="1261" y="0"></use><use href="#MJMATHI-66" x="2317" y="0"></use><use href="#MJMAIN-28" x="2867" y="0"></use><g transform="translate(3257,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1335" y="-213"></use></g><use href="#MJMAIN-22C5" x="4878" y="0"></use><g transform="translate(5378,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><use href="#MJMAIN-29" x="5951" y="0"></use></g><g transform="translate(0,626)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="748" y="-213"></use><use href="#MJMAIN-3D" x="1261" y="0"></use><use href="#MJMATHI-66" x="2317" y="0"></use><use href="#MJMAIN-28" x="2867" y="0"></use><g transform="translate(3257,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="1335" y="-213"></use></g><use href="#MJMAIN-22C5" x="4878" y="0"></use><g transform="translate(5378,0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="748" y="-213"></use></g><use href="#MJMAIN-29" x="6362" y="0"></use></g><g transform="translate(0,-676)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="748" y="-213"></use><use href="#MJMAIN-3D" x="1261" y="0"></use><use href="#MJMATHI-66" x="2317" y="0"></use><use href="#MJMAIN-28" x="2867" y="0"></use><g transform="translate(3257,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="1335" y="-213"></use></g><use href="#MJMAIN-22C5" x="4878" y="0"></use><g transform="translate(5378,0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="748" y="-213"></use></g><use href="#MJMAIN-29" x="6362" y="0"></use></g><g transform="translate(0,-1978)"><use href="#MJMATHI-79"></use><use href="#MJMAIN-20D7" x="543" y="7"></use><use href="#MJMAIN-3D" x="822" y="0"></use><use href="#MJMATHI-66" x="1878" y="0"></use><use href="#MJMAIN-28" x="2428" y="0"></use><g transform="translate(2818,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="1335" y="-213"></use></g><use href="#MJMAIN-22C5" x="4439" y="0"></use><g transform="translate(4939,0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-33" x="748" y="-213"></use></g><use href="#MJMAIN-29" x="5923" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,1928)"><g id="mjx-eqn-17"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,626)"><g id="mjx-eqn-18"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-676)"><g id="mjx-eqn-19"><use href="#MJMAIN-28"></use><use href="#MJMAIN-31" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1978)"><g id="mjx-eqn-20"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-50">
\begin{align}
&\vec{a}_1=f(W_1\centerdot\vec{x})\\
&\vec{a}_2=f(W_2\centerdot\vec{a}_1)\\
&\vec{a}_3=f(W_3\centerdot\vec{a}_2)\\
&\vec{y}=f(W_4\centerdot\vec{a}_3)\\
\end{align}
</script><p></p><p data-anchor-id="092q">这就是神经网络输出值的计算方法。</p><div class="md-section-divider"></div><h2 data-anchor-id="0454" id="神经网络的训练">神经网络的训练</h2><p data-anchor-id="lx3f">现在，我们需要知道一个神经网络的每个连接上的权值是如何得到的。我们可以说神经网络是一个<strong>模型</strong>，那么这些权值就是模型的<strong>参数</strong>，也就是模型要学习的东西。然而，一个神经网络的连接方式、网络的层数、每层的节点数这些参数，则不是学习出来的，而是人为事先设置的。对于这些人为设置的参数，我们称之为<strong>超参数(Hyper-Parameters)</strong>。</p><p data-anchor-id="crod">接下来，我们将要介绍神经网络的训练算法：反向传播算法。</p><div class="md-section-divider"></div><h3 data-anchor-id="io4w" id="反向传播算法back-propagation">反向传播算法(Back Propagation)</h3><p data-anchor-id="1rb4">我们首先直观的介绍反向传播算法，最后再来介绍这个算法的推导。当然读者也可以完全跳过推导部分，因为即使不知道如何推导，也不影响你写出来一个神经网络的训练代码。事实上，现在神经网络成熟的开源实现多如牛毛，除了练手之外，你可能都没有机会需要去写一个神经网络。</p><p data-anchor-id="vbpl">我们以<strong>监督学习</strong>为例来解释反向传播算法。在<a href="https://www.zybuluo.com/hanbingtao/note/448086" target="_blank">零基础入门深度学习(2) - 线性单元和梯度下降</a>一文中我们介绍了什么是<strong>监督学习</strong>，如果忘记了可以再看一下。另外，我们设神经元的激活函数<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-51-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -725.2000693679926 550.5 950.4001387359851" style="width: 1.331ex; height: 2.177ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-66"></use></g></svg></span><script type="math/tex" id="MathJax-Element-51">f</script>为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-52-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 3528.5 939.4001387359851" style="width: 8.226ex; height: 2.177ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-73"></use><use href="#MJMATHI-69" x="469" y="0"></use><use href="#MJMATHI-67" x="815" y="0"></use><use href="#MJMATHI-6D" x="1295" y="0"></use><use href="#MJMATHI-6F" x="2174" y="0"></use><use href="#MJMATHI-69" x="2659" y="0"></use><use href="#MJMATHI-64" x="3005" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-52">sigmoid</script>函数(不同激活函数的计算公式不同，详情见<a href="https://www.zybuluo.com/hanbingtao/note/476663#an1">反向传播算法的推导</a>一节)。</p><p data-anchor-id="h9o1">我们假设每个训练样本为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-53-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -924.324787109928 2350.9666666666667 1194.5248564779206" style="width: 5.444ex; height: 2.782ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-28"></use><g transform="translate(389,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><use href="#MJMAIN-2C" x="962" y="0"></use><g transform="translate(1407,0)"><use href="#MJMATHI-74" x="54" y="0"></use><use href="#MJMAIN-20D7" x="553" y="190"></use></g><use href="#MJMAIN-29" x="1961" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-53">(\vec{x},\vec{t})</script>，其中向量<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-54-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -740.324787109928 572.5 771.5248564779206" style="width: 1.331ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g></svg></span><script type="math/tex" id="MathJax-Element-54">\vec{x}</script>是训练样本的特征，而<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-55-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -924.324787109928 554.3 955.5248564779206" style="width: 1.331ex; height: 2.177ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-74" x="54" y="0"></use><use href="#MJMAIN-20D7" x="553" y="190"></use></g></svg></span><script type="math/tex" id="MathJax-Element-55">\vec{t}</script>是样本的目标值。</p><p data-anchor-id="yv1n"><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-6f27ced45cf5c0d8.png" alt=""></p><p data-anchor-id="ygu1">首先，我们根据上一节介绍的算法，用样本的特征<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-56-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -740.324787109928 572.5 771.5248564779206" style="width: 1.331ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g></svg></span><script type="math/tex" id="MathJax-Element-56">\vec{x}</script>，计算出神经网络中每个隐藏层节点的输出<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-57-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -461.20006936799257 873.8053928999522 638.8854201102238" style="width: 2.056ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="748" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-57">a_i</script>，以及输出层每个节点的输出<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-58-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 834.8053928999522 688.4001387359851" style="width: 1.935ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-58">y_i</script>。</p><p data-anchor-id="1s9t">然后，我们按照下面的方法计算出每个节点的误差项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-59-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 788.8053928999522 914.8854201102238" style="width: 1.815ex; height: 2.177ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="628" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-59">\delta_i</script>：</p><ul data-anchor-id="g1kw">
<li>对于输出层节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-60-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-60">i</script>，</li>
</ul><div class="md-section-divider"></div><p data-anchor-id="ks4m"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-61-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -904.613537109928 13950.301731524853 1288.3956226069529" style="width: 32.419ex; height: 3.024ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="628" y="-213"></use><use href="#MJMAIN-3D" x="1066" y="0"></use><g transform="translate(2122,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-28" x="2957" y="0"></use><use href="#MJMAIN-31" x="3347" y="0"></use><use href="#MJMAIN-2212" x="4069" y="0"></use><g transform="translate(5070,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="5905" y="0"></use><use href="#MJMAIN-28" x="6294" y="0"></use><g transform="translate(6684,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="511" y="-213"></use></g><use href="#MJMAIN-2212" x="7612" y="0"></use><g transform="translate(8613,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="9447" y="0"></use><use href="#MJMAIN-28" x="11837" y="0"></use><g transform="translate(12226,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-33" x="13060" y="0"></use><use href="#MJMAIN-29" x="13560" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-61">
\delta_i=y_i(1-y_i)(t_i-y_i)\qquad(式3)
</script><p></p><p data-anchor-id="7a0m">其中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-62-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 788.8053928999522 914.8854201102238" style="width: 1.815ex; height: 2.177ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="628" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-62">\delta_i</script>是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-63-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-63">i</script>的误差项，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-64-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 834.8053928999522 688.4001387359851" style="width: 1.935ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-64">y_i</script>是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-65-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-65">i</script>的<strong>输出值</strong>，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-66-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 705.8053928999522 823.8854201102238" style="width: 1.694ex; height: 1.935ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="511" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-66">t_i</script>是样本对应于节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-67-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-67">i</script>的<strong>目标值</strong>。举个例子，根据上图，对于输出层节点8来说，它的输出值是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-68-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 944.406943983867 688.4001387359851" style="width: 2.177ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-68">y_1</script>，而样本的目标值是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-69-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 815.406943983867 816.1072455171717" style="width: 1.935ex; height: 1.935ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-69">t_1</script>，带入上面的公式得到节点8的误差项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-70-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 898.406943983867 922.6635947032757" style="width: 2.056ex; height: 2.177ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-38" x="628" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-70">\delta_8</script>应该是：</p><div class="md-section-divider"></div><p data-anchor-id="ric6"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-71-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 10385.47916436378 1040.4001387359851" style="width: 24.073ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-38" x="628" y="-213"></use><use href="#MJMAIN-3D" x="1176" y="0"></use><g transform="translate(2232,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="693" y="-213"></use></g><use href="#MJMAIN-28" x="3176" y="0"></use><use href="#MJMAIN-31" x="3566" y="0"></use><use href="#MJMAIN-2212" x="4289" y="0"></use><g transform="translate(5289,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="6234" y="0"></use><use href="#MJMAIN-28" x="6623" y="0"></use><g transform="translate(7013,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="511" y="-213"></use></g><use href="#MJMAIN-2212" x="8050" y="0"></use><g transform="translate(9051,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="9995" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-71">
\delta_8=y_1(1-y_1)(t_1-y_1)
</script><p></p><ul data-anchor-id="4gyz">
<li>对于隐藏层节点，</li>
</ul><div class="md-section-divider"></div><p data-anchor-id="6dau"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-72-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -970.2000693679926 16354.838270739212 2336.391173992012" style="width: 37.984ex; height: 5.444ex; vertical-align: -3.266ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="628" y="-213"></use><use href="#MJMAIN-3D" x="1066" y="0"></use><g transform="translate(2122,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="748" y="-213"></use></g><use href="#MJMAIN-28" x="2996" y="0"></use><use href="#MJMAIN-31" x="3386" y="0"></use><use href="#MJMAIN-2212" x="4108" y="0"></use><g transform="translate(5109,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="748" y="-213"></use></g><use href="#MJMAIN-29" x="5983" y="0"></use><g transform="translate(6539,0)"><use href="#MJSZ2-2211" x="874" y="0"></use><g transform="translate(0,-1110)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="521" y="0"></use><g transform="translate(840,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-75" x="485" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="1058" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="1419" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-75" x="1923" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="2495" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-73" x="2857" y="0"></use></g></g></g><g transform="translate(9899,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="521" y="0"></use></g></g><g transform="translate(11328,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-213"></use></g><use href="#MJMAIN-28" x="14242" y="0"></use><g transform="translate(14631,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-34" x="15464" y="0"></use><use href="#MJMAIN-29" x="15965" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-72">
\delta_i=a_i(1-a_i)\sum_{k\in{outputs}}w_{ki}\delta_k\qquad(式4)
</script><p></p><p data-anchor-id="amd4">其中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-73-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -461.20006936799257 873.8053928999522 638.8854201102238" style="width: 2.056ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="748" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-73">a_i</script>是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-74-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-74">i</script>的输出值，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-75-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1429.5615792887368 640.8854201102238" style="width: 3.266ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="521" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-75">w_{ki}</script>是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-76-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-76">i</script>到它的下一层节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-77-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 521.5 745.4001387359851" style="width: 1.21ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6B"></use></g></svg></span><script type="math/tex" id="MathJax-Element-77">k</script>的连接的权重，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-78-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 913.2561863887845 914.8854201102238" style="width: 2.177ex; height: 2.177ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-78">\delta_k</script>是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-79-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-79">i</script>的下一层节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-80-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 521.5 745.4001387359851" style="width: 1.21ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6B"></use></g></svg></span><script type="math/tex" id="MathJax-Element-80">k</script>的误差项。例如，对于隐藏层节点4来说，计算方法如下：</p><div class="md-section-divider"></div><p data-anchor-id="g81d"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-81-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 13549.106940299249 1040.4001387359851" style="width: 31.452ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="628" y="-213"></use><use href="#MJMAIN-3D" x="1176" y="0"></use><g transform="translate(2232,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="748" y="-213"></use></g><use href="#MJMAIN-28" x="3215" y="0"></use><use href="#MJMAIN-31" x="3605" y="0"></use><use href="#MJMAIN-2212" x="4328" y="0"></use><g transform="translate(5328,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="748" y="-213"></use></g><use href="#MJMAIN-29" x="6312" y="0"></use><use href="#MJMAIN-28" x="6701" y="0"></use><g transform="translate(7091,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-38"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="500" y="0"></use></g></g><g transform="translate(8615,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-38" x="628" y="-213"></use></g><use href="#MJMAIN-2B" x="9736" y="0"></use><g transform="translate(10736,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-39"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="500" y="0"></use></g></g><g transform="translate(12261,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-39" x="628" y="-213"></use></g><use href="#MJMAIN-29" x="13159" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-81">
\delta_4=a_4(1-a_4)(w_{84}\delta_8+w_{94}\delta_9)
</script><p></p><p data-anchor-id="48s2">最后，更新每个连接上的权值：</p><div class="md-section-divider"></div><p data-anchor-id="f819"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-82-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -904.613537109928 12144.972690238306 1288.3956226069529" style="width: 28.185ex; height: 3.024ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g><use href="#MJMAIN-2190" x="1630" y="0"></use><g transform="translate(2908,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-2B" x="4483" y="0"></use><use href="#MJMATHI-3B7" x="5483" y="0"></use><g transform="translate(5987,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="628" y="-213"></use></g><g transform="translate(6823,0)"><use href="#MJMATHI-78"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-28" x="10032" y="0"></use><g transform="translate(10421,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-35" x="11254" y="0"></use><use href="#MJMAIN-29" x="11755" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-82">
w_{ji}\gets w_{ji}+\eta\delta_jx_{ji}\qquad(式5)
</script><p></p><p data-anchor-id="fnuj">其中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-83-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-83">w_{ji}</script>是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-84-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-84">i</script>到节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-85-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-11.5 -681.2000693679926 424 905.4001387359851" style="width: 0.968ex; height: 2.056ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6A"></use></g></svg></span><script type="math/tex" id="MathJax-Element-85">j</script>的权重，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-86-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 503.5 699.4001387359851" style="width: 1.21ex; height: 1.573ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B7"></use></g></svg></span><script type="math/tex" id="MathJax-Element-86">\eta</script>是一个成为<strong>学习速率</strong>的常数，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-87-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 836.1815472394509 1051.3570288792273" style="width: 1.935ex; height: 2.419ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="628" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-87">\delta_j</script>是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-88-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-11.5 -681.2000693679926 424 905.4001387359851" style="width: 0.968ex; height: 2.056ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6A"></use></g></svg></span><script type="math/tex" id="MathJax-Element-88">j</script>的误差项，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-89-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 1208.486940139403 776.3570288792274" style="width: 2.782ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-89">x_{ji}</script>是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-90-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-90">i</script>传递给节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-91-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-11.5 -681.2000693679926 424 905.4001387359851" style="width: 0.968ex; height: 2.056ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6A"></use></g></svg></span><script type="math/tex" id="MathJax-Element-91">j</script>的输入。例如，权重<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-92-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -697.2000693679926 1670.906943983867 882.6635947032757" style="width: 3.871ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-38" x="1013" y="-213"></use><use href="#MJMAIN-34" x="1170" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-92">w_84</script>的更新方法如下：</p><div class="md-section-divider"></div><p data-anchor-id="kmsn"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-93-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 8212.941663903202 973.4001387359851" style="width: 19.113ex; height: 2.298ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-38"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="500" y="0"></use></g><use href="#MJMAIN-2190" x="1802" y="0"></use><g transform="translate(3080,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-38"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="500" y="0"></use></g></g><use href="#MJMAIN-2B" x="4826" y="0"></use><use href="#MJMATHI-3B7" x="5827" y="0"></use><g transform="translate(6331,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-38" x="628" y="-213"></use></g><g transform="translate(7229,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="748" y="-213"></use></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-93">
w_{84}\gets w_{84}+\eta\delta_8 a_4
</script><p></p><p data-anchor-id="o3kn">类似的，权重<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-94-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1524.313887967734 633.1072455171717" style="width: 3.508ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-94">w_{41}</script>的更新方法如下：</p><div class="md-section-divider"></div><p data-anchor-id="5tk6"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-95-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 8255.941663903202 973.4001387359851" style="width: 19.234ex; height: 2.298ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g><use href="#MJMAIN-2190" x="1802" y="0"></use><g transform="translate(3080,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="500" y="0"></use></g></g><use href="#MJMAIN-2B" x="4826" y="0"></use><use href="#MJMATHI-3B7" x="5827" y="0"></use><g transform="translate(6331,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="628" y="-213"></use></g><g transform="translate(7229,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="809" y="-213"></use></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-95">
w_{41}\gets w_{41}+\eta\delta_4 x_1
</script><p></p><p data-anchor-id="fym2">偏置项的输入值永远为1。例如，节点4的偏置项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-96-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1474.1093065034893 640.8854201102238" style="width: 3.387ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-96">w_{4b}</script>应该按照下面的方法计算：</p><div class="md-section-divider"></div><p data-anchor-id="k0cb"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-97-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 7129.125556990846 973.4001387359851" style="width: 16.573ex; height: 2.298ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g><use href="#MJMAIN-2190" x="1751" y="0"></use><g transform="translate(3030,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-34"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="500" y="0"></use></g></g><use href="#MJMAIN-2B" x="4726" y="0"></use><use href="#MJMATHI-3B7" x="5727" y="0"></use><g transform="translate(6230,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="628" y="-213"></use></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-97">
w_{4b}\gets w_{4b}+\eta\delta_4
</script><p></p><p data-anchor-id="hdcf">我们已经介绍了神经网络每个节点误差项的计算和权重更新方法。显然，计算一个节点的误差项，需要先计算每个与其相连的下一层节点的误差项。这就要求误差项的计算顺序必须是从输出层开始，然后反向依次计算每个隐藏层的误差项，直到与输入层相连的那个隐藏层。这就是反向传播算法的名字的含义。当所有节点的误差项计算完毕后，我们就可以根据<strong>式5</strong>来更新所有的权重。</p><p data-anchor-id="6frf">以上就是基本的反向传播算法，并不是很复杂，您弄清楚了么？</p><div class="md-section-divider"></div><h3 data-anchor-id="mln1" id="反向传播算法的推导">反向传播算法的推导</h3><p data-anchor-id="xdxw">反向传播算法其实就是链式求导法则的应用。然而，这个如此简单且显而易见的方法，却是在Roseblatt提出感知器算法将近30年之后才被发明和普及的。对此，Bengio这样回应道：</p><blockquote data-anchor-id="2ymm" class="white-blockquote">
  <p>很多看似显而易见的想法只有在事后才变得显而易见。</p>
</blockquote><p data-anchor-id="hndg">接下来，我们用链式求导法则来推导反向传播算法，也就是上一小节的<strong>式3</strong>、<strong>式4</strong>、<strong>式5</strong>。</p><p data-anchor-id="9muo"><strong><em>前方高能预警——接下来是数学公式重灾区，读者可以酌情阅读，不必强求。</em></strong></p><p data-anchor-id="ub3y">按照机器学习的通用套路，我们先确定神经网络的目标函数，然后用<strong>随机梯度下降</strong>优化算法去求目标函数最小值时的参数值。</p><p data-anchor-id="qxcc">我们取网络所有输出层节点的误差平方和作为目标函数：</p><div class="md-section-divider"></div><p data-anchor-id="d5sd"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-98-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1362.7080693679925 10634.844673360622 2705.5646502128566" style="width: 24.677ex; height: 6.29ex; vertical-align: -3.266ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use><use href="#MJMAIN-2261" x="1486" y="0"></use><g transform="translate(2662,0)"><rect stroke="none" width="620" height="60" x="0" y="220"></rect><use href="#MJMAIN-31" x="60" y="676"></use><use href="#MJMAIN-32" x="60" y="-686"></use></g><g transform="translate(3569,0)"><use href="#MJSZ2-2211" x="811" y="0"></use><g transform="translate(0,-1086)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-69"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="345" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F" x="1013" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-75" x="1498" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="2071" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="2432" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-75" x="2936" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="3508" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-73" x="3870" y="0"></use></g></g><use href="#MJMAIN-28" x="6638" y="0"></use><g transform="translate(7027,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="511" y="-213"></use></g><use href="#MJMAIN-2212" x="7955" y="0"></use><g transform="translate(8956,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><g transform="translate(9791,0)"><use href="#MJMAIN-29"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="550" y="583"></use></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-98">
E_d\equiv\frac{1}{2}\sum_{i\in outputs}(t_i-y_i)^2
</script><p></p><p data-anchor-id="z1dj">其中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-99-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -700.2000693679926 1208.6703999511576 877.1783133290371" style="width: 2.782ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-99">E_d</script>表示是样本<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-100-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 523.5 744.4001387359851" style="width: 1.21ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-64"></use></g></svg></span><script type="math/tex" id="MathJax-Element-100">d</script>的误差。</p><p data-anchor-id="wydc">然后，我们用文章<a href="https://www.zybuluo.com/hanbingtao/note/448086" target="_blank">零基础入门深度学习(2) - 线性单元和梯度下降</a>中介绍的<strong>随机梯度下降</strong>算法对目标函数进行优化：</p><div class="md-section-divider"></div><p data-anchor-id="1b6r"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-101-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1411.7080693679925 8434.127487084877 2412.072891782453" style="width: 19.597ex; height: 5.565ex; vertical-align: -2.419ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g><use href="#MJMAIN-2190" x="1630" y="0"></use><g transform="translate(2908,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-2212" x="4483" y="0"></use><use href="#MJMATHI-3B7" x="5483" y="0"></use><g transform="translate(6274,0)"><rect stroke="none" width="2039" height="60" x="0" y="220"></rect><g transform="translate(131,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-101">
w_{ji}\gets w_{ji}-\eta\frac{\partial{E_d}}{\partial{w_{ji}}}
</script><p></p><p data-anchor-id="kmvo">随机梯度下降算法也就是需要求出误差<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-102-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -700.2000693679926 1208.6703999511576 877.1783133290371" style="width: 2.782ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-102">E_d</script>对于每个权重<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-103-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-103">w_{ji}</script>的偏导数（也就是梯度），怎么求呢？</p><p data-anchor-id="qrep"><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-6f27ced45cf5c0d8.png" alt="" title=""></p><p data-anchor-id="f48g">观察上图，我们发现权重<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-104-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-104">w_{ji}</script>仅能通过影响节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-105-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-11.5 -681.2000693679926 424 905.4001387359851" style="width: 0.968ex; height: 2.056ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6A"></use></g></svg></span><script type="math/tex" id="MathJax-Element-105">j</script>的输入值影响网络的其它部分，设<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-106-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1820.181547239451 960.3570288792274" style="width: 4.234ex; height: 2.177ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-106">net_j</script>是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-107-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-11.5 -681.2000693679926 424 905.4001387359851" style="width: 0.968ex; height: 2.056ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6A"></use></g></svg></span><script type="math/tex" id="MathJax-Element-107">j</script>的<strong>加权输入</strong>，即</p><div class="md-section-divider"></div><p data-anchor-id="cgqt"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-108-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -2162.368858570448 44270.673387096766 3824.737717140896" style="width: 102.823ex; height: 8.831ex; vertical-align: -3.992ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(18646,0)"><g transform="translate(-15,0)"><g transform="translate(0,1097)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g><g transform="translate(2083,0)"><g transform="translate(0,1097)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-77"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="1013" y="-213"></use><use href="#MJMAIN-2192" x="53" y="534"></use></g><use href="#MJMAIN-22C5" x="2386" y="0"></use><g transform="translate(2887,0)"><g transform="translate(18,0)"><use href="#MJMATHI-78"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="809" y="-213"></use></g><use href="#MJMAIN-2192" x="0" y="533"></use></g></g><g transform="translate(0,-449)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJSZ2-2211"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="848" y="-1536"></use></g><g transform="translate(2667,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><g transform="translate(4019,0)"><use href="#MJMATHI-78"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,1097)"><g id="mjx-eqn-21"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-449)"><g id="mjx-eqn-22"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-108">
\begin{align}
net_j&=\vec{w_j}\centerdot\vec{x_j}\\
&=\sum_{i}{w_{ji}}x_{ji}
\end{align}
</script><p></p><p data-anchor-id="432c"><span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-109-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -700.2000693679926 1208.6703999511576 877.1783133290371" style="width: 2.782ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-109">E_d</script>是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-110-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1820.181547239451 960.3570288792274" style="width: 4.234ex; height: 2.177ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-110">net_j</script>的函数，而<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-111-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1820.181547239451 960.3570288792274" style="width: 4.234ex; height: 2.177ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-111">net_j</script>是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-112-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-112">w_{ji}</script>的函数。根据链式求导法则，可以得到：</p><div class="md-section-divider"></div><p data-anchor-id="qjsy"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-113-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -4235.865951984163 44270.673387096766 7971.731903968325" style="width: 102.823ex; height: 18.508ex; vertical-align: -8.71ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(16517,0)"><g transform="translate(-15,0)"><g transform="translate(0,2735)"><g transform="translate(120,0)"><rect stroke="none" width="2039" height="60" x="0" y="220"></rect><g transform="translate(131,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g></g><g transform="translate(2543,0)"><g transform="translate(0,2735)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(365,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g><g transform="translate(4090,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(60,765)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g><g transform="translate(293,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g><g transform="translate(0,-62)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(365,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g><g transform="translate(4090,0)"><rect stroke="none" width="4815" height="60" x="0" y="220"></rect><g transform="translate(60,765)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJSZ1-2211"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="1494" y="-405"></use><g transform="translate(1567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><g transform="translate(2919,0)"><use href="#MJMATHI-78"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g><g transform="translate(1447,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g><g transform="translate(0,-2736)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(365,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g><g transform="translate(3970,0)"><use href="#MJMATHI-78"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,2735)"><g id="mjx-eqn-23"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-62)"><g id="mjx-eqn-24"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-2736)"><g id="mjx-eqn-25"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-113">
\begin{align}
\frac{\partial{E_d}}{\partial{w_{ji}}}&=\frac{\partial{E_d}}{\partial{net_j}}\frac{\partial{net_j}}{\partial{w_{ji}}}\\
&=\frac{\partial{E_d}}{\partial{net_j}}\frac{\partial{\sum_{i}{w_{ji}}x_{ji}}}{\partial{w_{ji}}}\\
&=\frac{\partial{E_d}}{\partial{net_j}}x_{ji}
\end{align}
</script> <br>
上式中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-114-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -462.20006936799257 1208.486940139403 776.3570288792274" style="width: 2.782ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-78"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-114">x_{ji}</script>是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-115-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-115">i</script>传递给节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-116-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-11.5 -681.2000693679926 424 905.4001387359851" style="width: 0.968ex; height: 2.056ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6A"></use></g></svg></span><script type="math/tex" id="MathJax-Element-116">j</script>的输入值，也就是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-117-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -681.2000693679926 345.5 712.4001387359851" style="width: 0.847ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-69"></use></g></svg></span><script type="math/tex" id="MathJax-Element-117">i</script>的输出值。<p></p><p data-anchor-id="5hl3">对于<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-118-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1044.710439052962 2048.3458133670038 1686.4208781059244" style="width: 4.718ex; height: 3.871ex; vertical-align: -1.573ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="1808" height="60" x="0" y="220"></rect><g transform="translate(276,519)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-45"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-414)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(754,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-118">\frac{\partial{E_d}}{\partial{net_j}}</script>的推导，需要区分<strong>输出层</strong>和<strong>隐藏层</strong>两种情况。</p><div class="md-section-divider"></div><h4 data-anchor-id="n0ie" id="输出层权值训练">输出层权值训练</h4><p data-anchor-id="zee1">对于<strong>输出层</strong>来说，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-119-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1820.181547239451 960.3570288792274" style="width: 4.234ex; height: 2.177ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-119">net_j</script>仅能通过节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-120-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-11.5 -681.2000693679926 424 905.4001387359851" style="width: 0.968ex; height: 2.056ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6A"></use></g></svg></span><script type="math/tex" id="MathJax-Element-120">j</script>的输出值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-121-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 882.1815472394509 777.3570288792274" style="width: 2.056ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-121">y_j</script>来影响网络其它部分，也就是说<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-122-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -700.2000693679926 1208.6703999511576 877.1783133290371" style="width: 2.782ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-122">E_d</script>是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-123-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 882.1815472394509 777.3570288792274" style="width: 2.056ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-123">y_j</script>的函数，而<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-124-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 882.1815472394509 777.3570288792274" style="width: 2.056ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-124">y_j</script>是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-125-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1820.181547239451 960.3570288792274" style="width: 4.234ex; height: 2.177ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-125">net_j</script>的函数，其中<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-126-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 8343.918650034459 1084.3570288792273" style="width: 19.355ex; height: 2.54ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use><use href="#MJMAIN-3D" x="1159" y="0"></use><use href="#MJMATHI-73" x="2216" y="0"></use><use href="#MJMATHI-69" x="2685" y="0"></use><use href="#MJMATHI-67" x="3031" y="0"></use><use href="#MJMATHI-6D" x="3511" y="0"></use><use href="#MJMATHI-6F" x="4390" y="0"></use><use href="#MJMATHI-69" x="4875" y="0"></use><use href="#MJMATHI-64" x="5221" y="0"></use><use href="#MJMAIN-28" x="5744" y="0"></use><use href="#MJMATHI-6E" x="6134" y="0"></use><use href="#MJMATHI-65" x="6734" y="0"></use><g transform="translate(7201,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g><use href="#MJMAIN-29" x="7954" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-126">y_j=sigmoid(net_j)</script>。所以我们可以再次使用链式求导法则：</p><div class="md-section-divider"></div><p data-anchor-id="dsw7"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-127-Frame" style="font-size: 100%; display: inline-block;"><span style="display: inline-block; white-space: nowrap; padding: 1px 0px;"><span style="display: inline-block; position: relative; width: 102.823ex; height: 5.685ex; vertical-align: -2.298ex;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1500.3648224144604 44270.673387096766 2500.7296448289208" style="width: 102.823ex; height: 5.806ex; position: absolute; bottom: 0ex; left: 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(17743,0)"><g transform="translate(-15,0)"><g transform="translate(120,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(365,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g><g transform="translate(3011,0)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="1896" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(223,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g></g></g><g transform="translate(3479,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(529,765)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g></g><g transform="translate(41690,0)"><g id="mjx-eqn-26"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></svg></span></span></span></div><script type="math/tex; mode=display" id="MathJax-Element-127">
\begin{align}
\frac{\partial{E_d}}{\partial{net_j}}&=\frac{\partial{E_d}}{\partial{y_j}}\frac{\partial{y_j}}{\partial{net_j}}\\
\end{align}
</script><p></p><p data-anchor-id="r2nd">考虑上式第一项:</p><div class="md-section-divider"></div><p data-anchor-id="b2sg"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-128-Frame" style="font-size: 100%; display: inline-block;"><span style="display: inline-block; white-space: nowrap; padding: 1px 0px;"><span style="display: inline-block; position: relative; width: 102.823ex; height: 15.605ex; vertical-align: -7.258ex;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -3637.0971467012832 44270.673387096766 6774.194293402567" style="width: 102.823ex; height: 15.726ex; position: absolute; bottom: 0ex; left: 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(15540,0)"><g transform="translate(-15,0)"><g transform="translate(0,2225)"><g transform="translate(120,0)"><rect stroke="none" width="1896" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(223,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g></g></g></g></g><g transform="translate(2399,0)"><g transform="translate(0,2225)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="1569" height="60" x="0" y="220"></rect><use href="#MJMAIN-2202" x="501" y="676"></use><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g></g></g><g transform="translate(3152,0)"><rect stroke="none" width="620" height="60" x="0" y="220"></rect><use href="#MJMAIN-31" x="60" y="676"></use><use href="#MJMAIN-32" x="60" y="-686"></use></g><g transform="translate(4059,0)"><use href="#MJSZ2-2211" x="811" y="0"></use><g transform="translate(0,-1086)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-69"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="345" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F" x="1013" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-75" x="1498" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="2071" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-70" x="2432" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-75" x="2936" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="3508" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-73" x="3870" y="0"></use></g></g><use href="#MJMAIN-28" x="7128" y="0"></use><g transform="translate(7517,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="511" y="-213"></use></g><use href="#MJMAIN-2212" x="8445" y="0"></use><g transform="translate(9446,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="693" y="-213"></use></g><g transform="translate(10281,0)"><use href="#MJMAIN-29"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="550" y="583"></use></g></g><g transform="translate(0,-791)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="1569" height="60" x="0" y="220"></rect><use href="#MJMAIN-2202" x="501" y="676"></use><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g></g></g><g transform="translate(3152,0)"><rect stroke="none" width="620" height="60" x="0" y="220"></rect><use href="#MJMAIN-31" x="60" y="676"></use><use href="#MJMAIN-32" x="60" y="-686"></use></g><use href="#MJMAIN-28" x="4059" y="0"></use><g transform="translate(4449,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g><use href="#MJMAIN-2212" x="5424" y="0"></use><g transform="translate(6425,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><g transform="translate(7307,0)"><use href="#MJMAIN-29"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="550" y="583"></use></g></g><g transform="translate(0,-2823)"><use href="#MJMAIN-3D"></use><use href="#MJMAIN-2212" x="1056" y="0"></use><use href="#MJMAIN-28" x="1834" y="0"></use><g transform="translate(2224,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g><use href="#MJMAIN-2212" x="3199" y="0"></use><g transform="translate(4200,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="5082" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,2225)"><g id="mjx-eqn-27"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-791)"><g id="mjx-eqn-28"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-2823)"><g id="mjx-eqn-29"><use href="#MJMAIN-28"></use><use href="#MJMAIN-32" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></span></span></div><script type="math/tex; mode=display" id="MathJax-Element-128">
\begin{align}
\frac{\partial{E_d}}{\partial{y_j}}&=\frac{\partial}{\partial{y_j}}\frac{1}{2}\sum_{i\in outputs}(t_i-y_i)^2\\
&=\frac{\partial}{\partial{y_j}}\frac{1}{2}(t_j-y_j)^2\\
&=-(t_j-y_j)
\end{align}
</script><p></p><p data-anchor-id="rvfo">考虑上式第二项：</p><div class="md-section-divider"></div><p data-anchor-id="u5gq"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-129-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -2190.8432674860815 44270.673387096766 3881.686534972163" style="width: 102.823ex; height: 9.073ex; vertical-align: -3.992ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(16740,0)"><g transform="translate(-15,0)"><g transform="translate(0,655)"><g transform="translate(120,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(529,765)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g></g><g transform="translate(3011,0)"><g transform="translate(0,655)"><use href="#MJMAIN-3D"></use><g transform="translate(1176,0)"><rect stroke="none" width="6815" height="60" x="0" y="220"></rect><g transform="translate(60,765)"><use href="#MJMAIN-2202"></use><use href="#MJMATHI-73" x="567" y="0"></use><use href="#MJMATHI-69" x="1037" y="0"></use><use href="#MJMATHI-67" x="1382" y="0"></use><use href="#MJMATHI-6D" x="1863" y="0"></use><use href="#MJMATHI-6F" x="2741" y="0"></use><use href="#MJMATHI-69" x="3227" y="0"></use><use href="#MJMATHI-64" x="3572" y="0"></use><use href="#MJMAIN-28" x="4096" y="0"></use><use href="#MJMATHI-6E" x="4485" y="0"></use><use href="#MJMATHI-65" x="5086" y="0"></use><g transform="translate(5552,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g><use href="#MJMAIN-29" x="6305" y="0"></use></g><g transform="translate(2213,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g><g transform="translate(0,-1377)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-28" x="1938" y="0"></use><use href="#MJMAIN-31" x="2327" y="0"></use><use href="#MJMAIN-2212" x="3050" y="0"></use><g transform="translate(4051,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="4933" y="0"></use></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,655)"><g id="mjx-eqn-30"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1377)"><g id="mjx-eqn-31"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-129">
\begin{align}
\frac{\partial{y_j}}{\partial{net_j}}&=\frac{\partial sigmoid(net_j)}{\partial{net_j}}\\
&=y_j(1-y_j)\\
\end{align}
</script><p></p><p data-anchor-id="r0w3">将第一项和第二项带入，得到：</p><div class="md-section-divider"></div><p data-anchor-id="xcif"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-130-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1411.7080693679925 12764.3521806417 2412.072891782453" style="width: 29.637ex; height: 5.565ex; vertical-align: -2.419ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(365,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g><use href="#MJMAIN-3D" x="3025" y="0"></use><use href="#MJMAIN-2212" x="4081" y="0"></use><use href="#MJMAIN-28" x="4860" y="0"></use><g transform="translate(5249,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g><use href="#MJMAIN-2212" x="6225" y="0"></use><g transform="translate(7225,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="8108" y="0"></use><g transform="translate(8497,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-28" x="9379" y="0"></use><use href="#MJMAIN-31" x="9769" y="0"></use><use href="#MJMAIN-2212" x="10491" y="0"></use><g transform="translate(11492,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="12374" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-130">
\frac{\partial{E_d}}{\partial{net_j}}=-(t_j-y_j)y_j(1-y_j)
</script><p></p><p data-anchor-id="l06o">如果令<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-131-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1044.710439052962 5163.7495828286765 1686.4208781059244" style="width: 11.976ex; height: 3.871ex; vertical-align: -1.573ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="628" y="-213"></use><use href="#MJMAIN-3D" x="1113" y="0"></use><use href="#MJMAIN-2212" x="2170" y="0"></use><g transform="translate(3235,0)"><rect stroke="none" width="1808" height="60" x="0" y="220"></rect><g transform="translate(276,519)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-45"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-414)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(754,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-131">\delta_j=-\frac{\partial{E_d}}{\partial{net_j}}</script>，也就是一个节点的误差项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-132-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 451.5 767.4001387359851" style="width: 1.089ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use></g></svg></span><script type="math/tex" id="MathJax-Element-132">\delta</script>是网络误差对这个节点输入的偏导数的相反数。带入上式，得到：</p><div class="md-section-divider"></div><p data-anchor-id="ta5e"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-133-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 10074.3521806417 1084.3570288792273" style="width: 23.347ex; height: 2.54ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="628" y="-213"></use><use href="#MJMAIN-3D" x="1113" y="0"></use><use href="#MJMAIN-28" x="2170" y="0"></use><g transform="translate(2559,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g><use href="#MJMAIN-2212" x="3535" y="0"></use><g transform="translate(4535,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="5418" y="0"></use><g transform="translate(5807,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-28" x="6689" y="0"></use><use href="#MJMAIN-31" x="7079" y="0"></use><use href="#MJMAIN-2212" x="7801" y="0"></use><g transform="translate(8802,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="9684" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-133">
\delta_j=(t_j-y_j)y_j(1-y_j)
</script><p></p><p data-anchor-id="js55">上式就是<strong>式3</strong>。</p><p data-anchor-id="q9f3">将上述推导带入随机梯度下降公式，得到：</p><div class="md-section-divider"></div><p data-anchor-id="0bkw"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-134-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -2801.993336034469 44270.673387096766 5103.986672068938" style="width: 102.823ex; height: 11.855ex; vertical-align: -5.444ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(14870,0)"><g transform="translate(-15,0)"><g transform="translate(0,1390)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g><g transform="translate(1616,0)"><g transform="translate(0,1390)"><use href="#MJMAIN-2190"></use><g transform="translate(1278,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-2212" x="2852" y="0"></use><use href="#MJMATHI-3B7" x="3853" y="0"></use><g transform="translate(4643,0)"><rect stroke="none" width="2039" height="60" x="0" y="220"></rect><g transform="translate(131,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g><g transform="translate(0,-642)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-2B" x="2630" y="0"></use><use href="#MJMATHI-3B7" x="3631" y="0"></use><use href="#MJMAIN-28" x="4135" y="0"></use><g transform="translate(4524,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g><use href="#MJMAIN-2212" x="5500" y="0"></use><g transform="translate(6500,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="7383" y="0"></use><g transform="translate(7772,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-28" x="8654" y="0"></use><use href="#MJMAIN-31" x="9044" y="0"></use><use href="#MJMAIN-2212" x="9766" y="0"></use><g transform="translate(10767,0)"><use href="#MJMATHI-79"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="693" y="-213"></use></g><use href="#MJMAIN-29" x="11649" y="0"></use><g transform="translate(12039,0)"><use href="#MJMATHI-78"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g><g transform="translate(0,-1988)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-2B" x="2630" y="0"></use><use href="#MJMATHI-3B7" x="3631" y="0"></use><g transform="translate(4135,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="628" y="-213"></use></g><g transform="translate(4971,0)"><use href="#MJMATHI-78"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,1390)"><g id="mjx-eqn-32"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-642)"><g id="mjx-eqn-33"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1988)"><g id="mjx-eqn-34"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-134">
\begin{align}
w_{ji}&\gets w_{ji}-\eta\frac{\partial{E_d}}{\partial{w_{ji}}}\\
&=w_{ji}+\eta(t_j-y_j)y_j(1-y_j)x_{ji}\\
&=w_{ji}+\eta\delta_jx_{ji}
\end{align}
</script><p></p><p data-anchor-id="c2kt">上式就是<strong>式5</strong>。</p><div class="md-section-divider"></div><h4 data-anchor-id="zkzc" id="隐藏层权值训练">隐藏层权值训练</h4><p data-anchor-id="wbn6">现在我们要推导出隐藏层的<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-135-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1044.710439052962 2048.3458133670038 1686.4208781059244" style="width: 4.718ex; height: 3.871ex; vertical-align: -1.573ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="1808" height="60" x="0" y="220"></rect><g transform="translate(276,519)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-45"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-414)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(754,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-135">\frac{\partial{E_d}}{\partial{net_j}}</script>。</p><p data-anchor-id="941o">首先，我们需要定义节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-136-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-11.5 -681.2000693679926 424 905.4001387359851" style="width: 0.968ex; height: 2.056ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6A"></use></g></svg></span><script type="math/tex" id="MathJax-Element-136">j</script>的所有直接下游节点的集合<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-137-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 6979.5 1040.4001387359851" style="width: 16.21ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-44"></use><use href="#MJMATHI-6F" x="828" y="0"></use><use href="#MJMATHI-77" x="1314" y="0"></use><use href="#MJMATHI-6E" x="2030" y="0"></use><use href="#MJMATHI-73" x="2631" y="0"></use><use href="#MJMATHI-74" x="3100" y="0"></use><use href="#MJMATHI-72" x="3462" y="0"></use><use href="#MJMATHI-65" x="3913" y="0"></use><use href="#MJMATHI-61" x="4380" y="0"></use><use href="#MJMATHI-6D" x="4909" y="0"></use><use href="#MJMAIN-28" x="5788" y="0"></use><use href="#MJMATHI-6A" x="6177" y="0"></use><use href="#MJMAIN-29" x="6590" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-137">Downstream(j)</script>。例如，对于节点4来说，它的直接下游节点是节点8、节点9。可以看到<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-138-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1820.181547239451 960.3570288792274" style="width: 4.234ex; height: 2.177ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-138">net_j</script>只能通过影响<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-139-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 6979.5 1040.4001387359851" style="width: 16.21ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-44"></use><use href="#MJMATHI-6F" x="828" y="0"></use><use href="#MJMATHI-77" x="1314" y="0"></use><use href="#MJMATHI-6E" x="2030" y="0"></use><use href="#MJMATHI-73" x="2631" y="0"></use><use href="#MJMATHI-74" x="3100" y="0"></use><use href="#MJMATHI-72" x="3462" y="0"></use><use href="#MJMATHI-65" x="3913" y="0"></use><use href="#MJMATHI-61" x="4380" y="0"></use><use href="#MJMATHI-6D" x="4909" y="0"></use><use href="#MJMAIN-28" x="5788" y="0"></use><use href="#MJMATHI-6A" x="6177" y="0"></use><use href="#MJMAIN-29" x="6590" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-139">Downstream(j)</script>再影响<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-140-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -700.2000693679926 1208.6703999511576 877.1783133290371" style="width: 2.782ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-140">E_d</script>。设<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-141-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1897.2561863887845 823.8854201102238" style="width: 4.355ex; height: 1.935ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-141">net_k</script>是节点<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-142-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-11.5 -681.2000693679926 424 905.4001387359851" style="width: 0.968ex; height: 2.056ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6A"></use></g></svg></span><script type="math/tex" id="MathJax-Element-142">j</script>的下游节点的输入，则<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-143-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -700.2000693679926 1208.6703999511576 877.1783133290371" style="width: 2.782ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-143">E_d</script>是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-144-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1897.2561863887845 823.8854201102238" style="width: 4.355ex; height: 1.935ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-144">net_k</script>的函数，而<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-145-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1897.2561863887845 823.8854201102238" style="width: 4.355ex; height: 1.935ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-145">net_k</script>是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-146-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1820.181547239451 960.3570288792274" style="width: 4.234ex; height: 2.177ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-146">net_j</script>的函数。因为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-147-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -646.2000693679926 1897.2561863887845 823.8854201102238" style="width: 4.355ex; height: 1.935ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="511" y="-213"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-147">net_k</script>有多个，我们应用全导数公式，可以做出如下推导：</p><div class="md-section-divider"></div><p data-anchor-id="yxfi"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-148-Frame" style="font-size: 100%; display: inline-block;"><span style="display: inline-block; white-space: nowrap; padding: 1px 0px;"><span style="display: inline-block; position: relative; width: 102.823ex; height: 41.129ex; vertical-align: -19.96ex;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -9122.433806661222 44270.673387096766 17744.867613322447" style="width: 102.823ex; height: 41.25ex; position: absolute; bottom: 0ex; left: 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(13457,0)"><g transform="translate(-15,0)"><g transform="translate(0,7710)"><g transform="translate(120,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(365,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g></g><g transform="translate(3011,0)"><g transform="translate(0,7710)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJSZ2-2211" x="2165" y="0"></use><g transform="translate(0,-1149)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="521" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-44" x="1189" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F" x="2017" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-77" x="2503" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="3219" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-73" x="3819" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="4289" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-72" x="4651" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="5102" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-61" x="5569" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="6098" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-28" x="6976" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="7366" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="7779" y="0"></use></g></g><g transform="translate(7118,0)"><rect stroke="none" width="2584" height="60" x="0" y="220"></rect><g transform="translate(404,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="511" y="-213"></use></g></g></g></g><g transform="translate(10110,0)"><rect stroke="none" width="2584" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="511" y="-213"></use></g></g></g><g transform="translate(98,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g><g transform="translate(0,4592)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJSZ2-2211" x="2165" y="0"></use><g transform="translate(0,-1149)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="521" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-44" x="1189" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F" x="2017" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-77" x="2503" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="3219" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-73" x="3819" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="4289" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-72" x="4651" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="5102" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-61" x="5569" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="6098" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-28" x="6976" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="7366" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="7779" y="0"></use></g></g><use href="#MJMAIN-2212" x="6998" y="0"></use><g transform="translate(7777,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-213"></use></g><g transform="translate(8977,0)"><rect stroke="none" width="2584" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="511" y="-213"></use></g></g></g><g transform="translate(98,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g><g transform="translate(0,1384)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJSZ2-2211" x="2165" y="0"></use><g transform="translate(0,-1149)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="521" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-44" x="1189" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F" x="2017" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-77" x="2503" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="3219" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-73" x="3819" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="4289" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-72" x="4651" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="5102" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-61" x="5569" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="6098" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-28" x="6976" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="7366" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="7779" y="0"></use></g></g><use href="#MJMAIN-2212" x="6998" y="0"></use><g transform="translate(7777,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-213"></use></g><g transform="translate(8977,0)"><rect stroke="none" width="2584" height="60" x="0" y="220"></rect><g transform="translate(60,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="511" y="-213"></use></g></g></g><g transform="translate(548,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="748" y="-213"></use></g></g></g><g transform="translate(11968,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(509,765)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="748" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g><g transform="translate(0,-1823)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJSZ2-2211" x="2165" y="0"></use><g transform="translate(0,-1149)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="521" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-44" x="1189" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F" x="2017" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-77" x="2503" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="3219" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-73" x="3819" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="4289" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-72" x="4651" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="5102" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-61" x="5569" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="6098" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-28" x="6976" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="7366" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="7779" y="0"></use></g></g><use href="#MJMAIN-2212" x="6998" y="0"></use><g transform="translate(7777,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-213"></use></g><g transform="translate(8690,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="521" y="0"></use></g></g><g transform="translate(10454,0)"><rect stroke="none" width="2507" height="60" x="0" y="220"></rect><g transform="translate(509,765)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="748" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(1067,0)"><use href="#MJMATHI-74"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g><g transform="translate(0,-4500)"><use href="#MJMAIN-3D"></use><g transform="translate(1056,0)"><use href="#MJSZ2-2211" x="2165" y="0"></use><g transform="translate(0,-1149)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="521" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-44" x="1189" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F" x="2017" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-77" x="2503" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="3219" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-73" x="3819" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="4289" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-72" x="4651" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="5102" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-61" x="5569" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="6098" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-28" x="6976" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="7366" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="7779" y="0"></use></g></g><use href="#MJMAIN-2212" x="6998" y="0"></use><g transform="translate(7777,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-213"></use></g><g transform="translate(8690,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="521" y="0"></use></g></g><g transform="translate(10167,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="748" y="-213"></use></g><use href="#MJMAIN-28" x="11088" y="0"></use><use href="#MJMAIN-31" x="11478" y="0"></use><use href="#MJMAIN-2212" x="12201" y="0"></use><g transform="translate(13201,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="748" y="-213"></use></g><use href="#MJMAIN-29" x="14122" y="0"></use></g><g transform="translate(0,-7178)"><use href="#MJMAIN-3D"></use><use href="#MJMAIN-2212" x="1056" y="0"></use><g transform="translate(1834,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="748" y="-213"></use></g><use href="#MJMAIN-28" x="2755" y="0"></use><use href="#MJMAIN-31" x="3145" y="0"></use><use href="#MJMAIN-2212" x="3868" y="0"></use><g transform="translate(4868,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="748" y="-213"></use></g><use href="#MJMAIN-29" x="5790" y="0"></use><g transform="translate(6346,0)"><use href="#MJSZ2-2211" x="2165" y="0"></use><g transform="translate(0,-1149)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="521" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-44" x="1189" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F" x="2017" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-77" x="2503" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="3219" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-73" x="3819" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="4289" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-72" x="4651" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="5102" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-61" x="5569" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="6098" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-28" x="6976" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="7366" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="7779" y="0"></use></g></g><g transform="translate(12288,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-213"></use></g><g transform="translate(13202,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="521" y="0"></use></g></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,7710)"><g id="mjx-eqn-35"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,4592)"><g id="mjx-eqn-36"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,1384)"><g id="mjx-eqn-37"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1823)"><g id="mjx-eqn-38"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-38" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-4500)"><g id="mjx-eqn-39"><use href="#MJMAIN-28"></use><use href="#MJMAIN-33" x="389" y="0"></use><use href="#MJMAIN-39" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-7178)"><g id="mjx-eqn-40"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-30" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></span></span></div><script type="math/tex; mode=display" id="MathJax-Element-148">
\begin{align}
\frac{\partial{E_d}}{\partial{net_j}}&=\sum_{k\in Downstream(j)}\frac{\partial{E_d}}{\partial{net_k}}\frac{\partial{net_k}}{\partial{net_j}}\\
&=\sum_{k\in Downstream(j)}-\delta_k\frac{\partial{net_k}}{\partial{net_j}}\\
&=\sum_{k\in Downstream(j)}-\delta_k\frac{\partial{net_k}}{\partial{a_j}}\frac{\partial{a_j}}{\partial{net_j}}\\
&=\sum_{k\in Downstream(j)}-\delta_kw_{kj}\frac{\partial{a_j}}{\partial{net_j}}\\
&=\sum_{k\in Downstream(j)}-\delta_kw_{kj}a_j(1-a_j)\\
&=-a_j(1-a_j)\sum_{k\in Downstream(j)}\delta_kw_{kj}
\end{align}
</script><p></p><p data-anchor-id="86e9">因为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-149-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1044.710439052962 5163.7495828286765 1686.4208781059244" style="width: 11.976ex; height: 3.871ex; vertical-align: -1.573ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="628" y="-213"></use><use href="#MJMAIN-3D" x="1113" y="0"></use><use href="#MJMAIN-2212" x="2170" y="0"></use><g transform="translate(3235,0)"><rect stroke="none" width="1808" height="60" x="0" y="220"></rect><g transform="translate(276,519)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-45"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-414)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="600" y="0"></use><g transform="translate(754,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-74"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A" x="511" y="-213"></use></g></g></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-149">\delta_j=-\frac{\partial{E_d}}{\partial{net_j}}</script>，带入上式得到：</p><div class="md-section-divider"></div><p data-anchor-id="qd5f"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-150-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -970.2000693679926 15014.573637191019 2415.5871334849053" style="width: 34.839ex; height: 5.565ex; vertical-align: -3.387ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="628" y="-213"></use><use href="#MJMAIN-3D" x="1113" y="0"></use><g transform="translate(2170,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="748" y="-213"></use></g><use href="#MJMAIN-28" x="3091" y="0"></use><use href="#MJMAIN-31" x="3480" y="0"></use><use href="#MJMAIN-2212" x="4203" y="0"></use><g transform="translate(5204,0)"><use href="#MJMATHI-61"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="748" y="-213"></use></g><use href="#MJMAIN-29" x="6125" y="0"></use><g transform="translate(6681,0)"><use href="#MJSZ2-2211" x="2165" y="0"></use><g transform="translate(0,-1149)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2208" x="521" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-44" x="1189" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6F" x="2017" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-77" x="2503" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6E" x="3219" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-73" x="3819" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-74" x="4289" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-72" x="4651" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-65" x="5102" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-61" x="5569" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6D" x="6098" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-28" x="6976" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="7366" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="7779" y="0"></use></g></g><g transform="translate(12624,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B" x="628" y="-213"></use></g><g transform="translate(13537,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6B"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="521" y="0"></use></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-150">
\delta_j=a_j(1-a_j)\sum_{k\in Downstream(j)}\delta_kw_{kj}
</script><p></p><p data-anchor-id="714y">上式就是<strong>式4</strong>。</p><p data-anchor-id="cmhp"><strong><em>——数学公式警报解除——</em></strong></p><p data-anchor-id="jqv3">至此，我们已经推导出了反向传播算法。需要注意的是，我们刚刚推导出的训练规则是根据激活函数是sigmoid函数、平方和误差、全连接网络、随机梯度下降优化算法。如果激活函数不同、误差计算方式不同、网络连接结构不同、优化算法不同，则具体的训练规则也会不一样。但是无论怎样，训练规则的推导方式都是一样的，应用链式求导法则进行推导即可。</p><div class="md-section-divider"></div><h2 data-anchor-id="f8dg" id="神经网络的实现">神经网络的实现</h2><blockquote data-anchor-id="4vka" class="white-blockquote">
  <p>完整代码请参考GitHub: <a href="https://github.com/hanbt/learn_dl/blob/master/bp.py" target="_blank">https://github.com/hanbt/learn_dl/blob/master/bp.py</a> (python2.7)</p>
</blockquote><p data-anchor-id="fpq5">现在，我们要根据前面的算法，实现一个基本的全连接神经网络，这并不需要太多代码。我们在这里依然采用面向对象设计。</p><p data-anchor-id="rduh">首先，我们先做一个基本的模型：</p><p data-anchor-id="5fb1"><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-2fbae2ee722fbef9.png" alt=""></p><p data-anchor-id="zxn4">如上图，可以分解出5个领域对象来实现神经网络：</p><ul data-anchor-id="om70">
<li><em>Network</em> 神经网络对象，提供API接口。它由若干层对象组成以及连接对象组成。</li>
<li><em>Layer</em> 层对象，由多个节点组成。</li>
<li><em>Node</em> 节点对象计算和记录节点自身的信息(比如输出值<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-151-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -461.20006936799257 529.5 491.40013873598514" style="width: 1.21ex; height: 1.089ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use></g></svg></span><script type="math/tex" id="MathJax-Element-151">a</script>、误差项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-152-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -737.2000693679926 451.5 767.4001387359851" style="width: 1.089ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use></g></svg></span><script type="math/tex" id="MathJax-Element-152">\delta</script>等)，以及与这个节点相关的上下游的连接。</li>
<li><em>Connection</em> 每个连接对象都要记录该连接的权重。</li>
<li><em>Connections</em> 仅仅作为Connection的集合对象，提供一些集合操作。</li>
</ul><p data-anchor-id="uvcw">Node实现如下：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="27j9" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="com"># 节点类，负责记录和维护节点自身信息以及与这个节点相关的上下游连接，实现输出值和误差项的计算。</span></code></li><li class="L1"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">Node</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> layer_index</span><span class="pun">,</span><span class="pln"> node_index</span><span class="pun">):</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L4"><code class="language-python"><span class="str">        构造节点对象。</span></code></li><li class="L5"><code class="language-python"><span class="str">        layer_index: 节点所属的层的编号</span></code></li><li class="L6"><code class="language-python"><span class="str">        node_index: 节点的编号</span></code></li><li class="L7"><code class="language-python"><span class="str">        '''</span></code></li><li class="L8"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">layer_index </span><span class="pun">=</span><span class="pln"> layer_index</span></code></li><li class="L9"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">node_index </span><span class="pun">=</span><span class="pln"> node_index</span></code></li><li class="L0"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">downstream </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L1"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">upstream </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L2"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">output </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span></code></li><li class="L3"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">delta </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span></code></li><li class="L4"><code class="language-python"></code></li><li class="L5"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> set_output</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> output</span><span class="pun">):</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L7"><code class="language-python"><span class="str">        设置节点的输出值。如果节点属于输入层会用到这个函数。</span></code></li><li class="L8"><code class="language-python"><span class="str">        '''</span></code></li><li class="L9"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">output </span><span class="pun">=</span><span class="pln"> output</span></code></li><li class="L0"><code class="language-python"></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> append_downstream_connection</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> conn</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        添加一个到下游节点的连接</span></code></li><li class="L4"><code class="language-python"><span class="str">        '''</span></code></li><li class="L5"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">downstream</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">conn</span><span class="pun">)</span></code></li><li class="L6"><code class="language-python"></code></li><li class="L7"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> append_upstream_connection</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> conn</span><span class="pun">):</span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L9"><code class="language-python"><span class="str">        添加一个到上游节点的连接</span></code></li><li class="L0"><code class="language-python"><span class="str">        '''</span></code></li><li class="L1"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">upstream</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">conn</span><span class="pun">)</span></code></li><li class="L2"><code class="language-python"></code></li><li class="L3"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_output</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L4"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L5"><code class="language-python"><span class="str">        根据式1计算节点的输出</span></code></li><li class="L6"><code class="language-python"><span class="str">        '''</span></code></li><li class="L7"><code class="language-python"><span class="pln">        output </span><span class="pun">=</span><span class="pln"> reduce</span><span class="pun">(</span><span class="kwd">lambda</span><span class="pln"> ret</span><span class="pun">,</span><span class="pln"> conn</span><span class="pun">:</span><span class="pln"> ret </span><span class="pun">+</span><span class="pln"> conn</span><span class="pun">.</span><span class="pln">upstream_node</span><span class="pun">.</span><span class="pln">output </span><span class="pun">*</span><span class="pln"> conn</span><span class="pun">.</span><span class="pln">weight</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">upstream</span><span class="pun">,</span><span class="pln"> </span><span class="lit">0</span><span class="pun">)</span></code></li><li class="L8"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">output </span><span class="pun">=</span><span class="pln"> sigmoid</span><span class="pun">(</span><span class="pln">output</span><span class="pun">)</span></code></li><li class="L9"><code class="language-python"></code></li><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_hidden_layer_delta</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        节点属于隐藏层时，根据式4计算delta</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        downstream_delta </span><span class="pun">=</span><span class="pln"> reduce</span><span class="pun">(</span></code></li><li class="L5"><code class="language-python"><span class="pln">            </span><span class="kwd">lambda</span><span class="pln"> ret</span><span class="pun">,</span><span class="pln"> conn</span><span class="pun">:</span><span class="pln"> ret </span><span class="pun">+</span><span class="pln"> conn</span><span class="pun">.</span><span class="pln">downstream_node</span><span class="pun">.</span><span class="pln">delta </span><span class="pun">*</span><span class="pln"> conn</span><span class="pun">.</span><span class="pln">weight</span><span class="pun">,</span></code></li><li class="L6"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">downstream</span><span class="pun">,</span><span class="pln"> </span><span class="lit">0.0</span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">delta </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">output </span><span class="pun">*</span><span class="pln"> </span><span class="pun">(</span><span class="lit">1</span><span class="pln"> </span><span class="pun">-</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">output</span><span class="pun">)</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> downstream_delta</span></code></li><li class="L8"><code class="language-python"></code></li><li class="L9"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_output_layer_delta</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> label</span><span class="pun">):</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L1"><code class="language-python"><span class="str">        节点属于输出层时，根据式3计算delta</span></code></li><li class="L2"><code class="language-python"><span class="str">        '''</span></code></li><li class="L3"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">delta </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">output </span><span class="pun">*</span><span class="pln"> </span><span class="pun">(</span><span class="lit">1</span><span class="pln"> </span><span class="pun">-</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">output</span><span class="pun">)</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> </span><span class="pun">(</span><span class="pln">label </span><span class="pun">-</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">output</span><span class="pun">)</span></code></li><li class="L4"><code class="language-python"></code></li><li class="L5"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __str__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L7"><code class="language-python"><span class="str">        打印节点的信息</span></code></li><li class="L8"><code class="language-python"><span class="str">        '''</span></code></li><li class="L9"><code class="language-python"><span class="pln">        node_str </span><span class="pun">=</span><span class="pln"> </span><span class="str">'%u-%u: output: %f delta: %f'</span><span class="pln"> </span><span class="pun">%</span><span class="pln"> </span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">layer_index</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">node_index</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">output</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">delta</span><span class="pun">)</span></code></li><li class="L0"><code class="language-python"><span class="pln">        downstream_str </span><span class="pun">=</span><span class="pln"> reduce</span><span class="pun">(</span><span class="kwd">lambda</span><span class="pln"> ret</span><span class="pun">,</span><span class="pln"> conn</span><span class="pun">:</span><span class="pln"> ret </span><span class="pun">+</span><span class="pln"> </span><span class="str">'\n\t'</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> str</span><span class="pun">(</span><span class="pln">conn</span><span class="pun">),</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">downstream</span><span class="pun">,</span><span class="pln"> </span><span class="str">''</span><span class="pun">)</span></code></li><li class="L1"><code class="language-python"><span class="pln">        upstream_str </span><span class="pun">=</span><span class="pln"> reduce</span><span class="pun">(</span><span class="kwd">lambda</span><span class="pln"> ret</span><span class="pun">,</span><span class="pln"> conn</span><span class="pun">:</span><span class="pln"> ret </span><span class="pun">+</span><span class="pln"> </span><span class="str">'\n\t'</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> str</span><span class="pun">(</span><span class="pln">conn</span><span class="pun">),</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">upstream</span><span class="pun">,</span><span class="pln"> </span><span class="str">''</span><span class="pun">)</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> node_str </span><span class="pun">+</span><span class="pln"> </span><span class="str">'\n\tdownstream:'</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> downstream_str </span><span class="pun">+</span><span class="pln"> </span><span class="str">'\n\tupstream:'</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> upstream_str </span></code></li></ol></pre><p data-anchor-id="4gwx">ConstNode对象，为了实现一个输出恒为1的节点(计算偏置项<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-153-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1120.202362519622 640.8854201102238" style="width: 2.661ex; height: 1.452ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-62" x="1013" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-153">w_b</script>时需要)</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="y073" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">ConstNode</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> layer_index</span><span class="pun">,</span><span class="pln"> node_index</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        构造节点对象。</span></code></li><li class="L4"><code class="language-python"><span class="str">        layer_index: 节点所属的层的编号</span></code></li><li class="L5"><code class="language-python"><span class="str">        node_index: 节点的编号</span></code></li><li class="L6"><code class="language-python"><span class="str">        '''</span><span class="pln">    </span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">layer_index </span><span class="pun">=</span><span class="pln"> layer_index</span></code></li><li class="L8"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">node_index </span><span class="pun">=</span><span class="pln"> node_index</span></code></li><li class="L9"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">downstream </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L0"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">output </span><span class="pun">=</span><span class="pln"> </span><span class="lit">1</span></code></li><li class="L1"><code class="language-python"></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> append_downstream_connection</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> conn</span><span class="pun">):</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L4"><code class="language-python"><span class="str">        添加一个到下游节点的连接</span></code></li><li class="L5"><code class="language-python"><span class="str">        '''</span><span class="pln">       </span></code></li><li class="L6"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">downstream</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">conn</span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"></code></li><li class="L8"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_hidden_layer_delta</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L9"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L0"><code class="language-python"><span class="str">        节点属于隐藏层时，根据式4计算delta</span></code></li><li class="L1"><code class="language-python"><span class="str">        '''</span></code></li><li class="L2"><code class="language-python"><span class="pln">        downstream_delta </span><span class="pun">=</span><span class="pln"> reduce</span><span class="pun">(</span></code></li><li class="L3"><code class="language-python"><span class="pln">            </span><span class="kwd">lambda</span><span class="pln"> ret</span><span class="pun">,</span><span class="pln"> conn</span><span class="pun">:</span><span class="pln"> ret </span><span class="pun">+</span><span class="pln"> conn</span><span class="pun">.</span><span class="pln">downstream_node</span><span class="pun">.</span><span class="pln">delta </span><span class="pun">*</span><span class="pln"> conn</span><span class="pun">.</span><span class="pln">weight</span><span class="pun">,</span></code></li><li class="L4"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">downstream</span><span class="pun">,</span><span class="pln"> </span><span class="lit">0.0</span><span class="pun">)</span></code></li><li class="L5"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">delta </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">output </span><span class="pun">*</span><span class="pln"> </span><span class="pun">(</span><span class="lit">1</span><span class="pln"> </span><span class="pun">-</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">output</span><span class="pun">)</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> downstream_delta</span></code></li><li class="L6"><code class="language-python"></code></li><li class="L7"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __str__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L9"><code class="language-python"><span class="str">        打印节点的信息</span></code></li><li class="L0"><code class="language-python"><span class="str">        '''</span></code></li><li class="L1"><code class="language-python"><span class="pln">        node_str </span><span class="pun">=</span><span class="pln"> </span><span class="str">'%u-%u: output: 1'</span><span class="pln"> </span><span class="pun">%</span><span class="pln"> </span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">layer_index</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">node_index</span><span class="pun">)</span></code></li><li class="L2"><code class="language-python"><span class="pln">        downstream_str </span><span class="pun">=</span><span class="pln"> reduce</span><span class="pun">(</span><span class="kwd">lambda</span><span class="pln"> ret</span><span class="pun">,</span><span class="pln"> conn</span><span class="pun">:</span><span class="pln"> ret </span><span class="pun">+</span><span class="pln"> </span><span class="str">'\n\t'</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> str</span><span class="pun">(</span><span class="pln">conn</span><span class="pun">),</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">downstream</span><span class="pun">,</span><span class="pln"> </span><span class="str">''</span><span class="pun">)</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> node_str </span><span class="pun">+</span><span class="pln"> </span><span class="str">'\n\tdownstream:'</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> downstream_str</span></code></li></ol></pre><p data-anchor-id="hue9">Layer对象，负责初始化一层。此外，作为Node的集合对象，提供对Node集合的操作。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="b3tz" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">Layer</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> layer_index</span><span class="pun">,</span><span class="pln"> node_count</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        初始化一层</span></code></li><li class="L4"><code class="language-python"><span class="str">        layer_index: 层编号</span></code></li><li class="L5"><code class="language-python"><span class="str">        node_count: 层所包含的节点个数</span></code></li><li class="L6"><code class="language-python"><span class="str">        '''</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">layer_index </span><span class="pun">=</span><span class="pln"> layer_index</span></code></li><li class="L8"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">nodes </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L9"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">node_count</span><span class="pun">):</span></code></li><li class="L0"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">nodes</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="typ">Node</span><span class="pun">(</span><span class="pln">layer_index</span><span class="pun">,</span><span class="pln"> i</span><span class="pun">))</span></code></li><li class="L1"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">nodes</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="typ">ConstNode</span><span class="pun">(</span><span class="pln">layer_index</span><span class="pun">,</span><span class="pln"> node_count</span><span class="pun">))</span></code></li><li class="L2"><code class="language-python"></code></li><li class="L3"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> set_output</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> data</span><span class="pun">):</span></code></li><li class="L4"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L5"><code class="language-python"><span class="str">        设置层的输出。当层是输入层时会用到。</span></code></li><li class="L6"><code class="language-python"><span class="str">        '''</span></code></li><li class="L7"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">len</span><span class="pun">(</span><span class="pln">data</span><span class="pun">)):</span></code></li><li class="L8"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">nodes</span><span class="pun">[</span><span class="pln">i</span><span class="pun">].</span><span class="pln">set_output</span><span class="pun">(</span><span class="pln">data</span><span class="pun">[</span><span class="pln">i</span><span class="pun">])</span></code></li><li class="L9"><code class="language-python"></code></li><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_output</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        计算层的输出向量</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> node </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">nodes</span><span class="pun">[:-</span><span class="lit">1</span><span class="pun">]:</span></code></li><li class="L5"><code class="language-python"><span class="pln">            node</span><span class="pun">.</span><span class="pln">calc_output</span><span class="pun">()</span></code></li><li class="L6"><code class="language-python"></code></li><li class="L7"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> dump</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L9"><code class="language-python"><span class="str">        打印层的信息</span></code></li><li class="L0"><code class="language-python"><span class="str">        '''</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> node </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">nodes</span><span class="pun">:</span></code></li><li class="L2"><code class="language-python"><span class="pln">            </span><span class="kwd">print</span><span class="pln"> node</span></code></li></ol></pre><p data-anchor-id="xwmo">Connection对象，主要职责是记录连接的权重，以及这个连接所关联的上下游节点。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="qxd8" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">Connection</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> upstream_node</span><span class="pun">,</span><span class="pln"> downstream_node</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        初始化连接，权重初始化为是一个很小的随机数</span></code></li><li class="L4"><code class="language-python"><span class="str">        upstream_node: 连接的上游节点</span></code></li><li class="L5"><code class="language-python"><span class="str">        downstream_node: 连接的下游节点</span></code></li><li class="L6"><code class="language-python"><span class="str">        '''</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">upstream_node </span><span class="pun">=</span><span class="pln"> upstream_node</span></code></li><li class="L8"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">downstream_node </span><span class="pun">=</span><span class="pln"> downstream_node</span></code></li><li class="L9"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">weight </span><span class="pun">=</span><span class="pln"> random</span><span class="pun">.</span><span class="pln">uniform</span><span class="pun">(-</span><span class="lit">0.1</span><span class="pun">,</span><span class="pln"> </span><span class="lit">0.1</span><span class="pun">)</span></code></li><li class="L0"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">gradient </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0.0</span></code></li><li class="L1"><code class="language-python"></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_gradient</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L4"><code class="language-python"><span class="str">        计算梯度</span></code></li><li class="L5"><code class="language-python"><span class="str">        '''</span></code></li><li class="L6"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">gradient </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">downstream_node</span><span class="pun">.</span><span class="pln">delta </span><span class="pun">*</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">upstream_node</span><span class="pun">.</span><span class="pln">output</span></code></li><li class="L7"><code class="language-python"></code></li><li class="L8"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> get_gradient</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L9"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L0"><code class="language-python"><span class="str">        获取当前的梯度</span></code></li><li class="L1"><code class="language-python"><span class="str">        '''</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">gradient</span></code></li><li class="L3"><code class="language-python"></code></li><li class="L4"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> update_weight</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> rate</span><span class="pun">):</span></code></li><li class="L5"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L6"><code class="language-python"><span class="str">        根据梯度下降算法更新权重</span></code></li><li class="L7"><code class="language-python"><span class="str">        '''</span></code></li><li class="L8"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">calc_gradient</span><span class="pun">()</span></code></li><li class="L9"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">weight </span><span class="pun">+=</span><span class="pln"> rate </span><span class="pun">*</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">gradient</span></code></li><li class="L0"><code class="language-python"></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __str__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        打印连接信息</span></code></li><li class="L4"><code class="language-python"><span class="str">        '''</span></code></li><li class="L5"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> </span><span class="str">'(%u-%u) -&gt; (%u-%u) = %f'</span><span class="pln"> </span><span class="pun">%</span><span class="pln"> </span><span class="pun">(</span></code></li><li class="L6"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">upstream_node</span><span class="pun">.</span><span class="pln">layer_index</span><span class="pun">,</span><span class="pln"> </span></code></li><li class="L7"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">upstream_node</span><span class="pun">.</span><span class="pln">node_index</span><span class="pun">,</span></code></li><li class="L8"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">downstream_node</span><span class="pun">.</span><span class="pln">layer_index</span><span class="pun">,</span><span class="pln"> </span></code></li><li class="L9"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">downstream_node</span><span class="pun">.</span><span class="pln">node_index</span><span class="pun">,</span><span class="pln"> </span></code></li><li class="L0"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">weight</span><span class="pun">)</span></code></li></ol></pre><p data-anchor-id="mdf6">Connections对象，提供Connection集合操作。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="xha7" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">Connections</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">connections </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L3"><code class="language-python"></code></li><li class="L4"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> add_connection</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> connection</span><span class="pun">):</span></code></li><li class="L5"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">connections</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">connection</span><span class="pun">)</span></code></li><li class="L6"><code class="language-python"></code></li><li class="L7"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> dump</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> conn </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">connections</span><span class="pun">:</span></code></li><li class="L9"><code class="language-python"><span class="pln">            </span><span class="kwd">print</span><span class="pln"> conn</span></code></li></ol></pre><p data-anchor-id="fbfg">Network对象，提供API。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="03nu" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">Network</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> layers</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        初始化一个全连接神经网络</span></code></li><li class="L4"><code class="language-python"><span class="str">        layers: 二维数组，描述神经网络每层节点数</span></code></li><li class="L5"><code class="language-python"><span class="str">        '''</span></code></li><li class="L6"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">connections </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Connections</span><span class="pun">()</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">layers </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L8"><code class="language-python"><span class="pln">        layer_count </span><span class="pun">=</span><span class="pln"> len</span><span class="pun">(</span><span class="pln">layers</span><span class="pun">)</span></code></li><li class="L9"><code class="language-python"><span class="pln">        node_count </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">layer_count</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="typ">Layer</span><span class="pun">(</span><span class="pln">i</span><span class="pun">,</span><span class="pln"> layers</span><span class="pun">[</span><span class="pln">i</span><span class="pun">]))</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> layer </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">layer_count </span><span class="pun">-</span><span class="pln"> </span><span class="lit">1</span><span class="pun">):</span></code></li><li class="L3"><code class="language-python"><span class="pln">            connections </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span><span class="typ">Connection</span><span class="pun">(</span><span class="pln">upstream_node</span><span class="pun">,</span><span class="pln"> downstream_node</span><span class="pun">)</span><span class="pln"> </span></code></li><li class="L4"><code class="language-python"><span class="pln">                           </span><span class="kwd">for</span><span class="pln"> upstream_node </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[</span><span class="pln">layer</span><span class="pun">].</span><span class="pln">nodes</span></code></li><li class="L5"><code class="language-python"><span class="pln">                           </span><span class="kwd">for</span><span class="pln"> downstream_node </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[</span><span class="pln">layer </span><span class="pun">+</span><span class="pln"> </span><span class="lit">1</span><span class="pun">].</span><span class="pln">nodes</span><span class="pun">[:-</span><span class="lit">1</span><span class="pun">]]</span></code></li><li class="L6"><code class="language-python"><span class="pln">            </span><span class="kwd">for</span><span class="pln"> conn </span><span class="kwd">in</span><span class="pln"> connections</span><span class="pun">:</span></code></li><li class="L7"><code class="language-python"><span class="pln">                self</span><span class="pun">.</span><span class="pln">connections</span><span class="pun">.</span><span class="pln">add_connection</span><span class="pun">(</span><span class="pln">conn</span><span class="pun">)</span></code></li><li class="L8"><code class="language-python"><span class="pln">                conn</span><span class="pun">.</span><span class="pln">downstream_node</span><span class="pun">.</span><span class="pln">append_upstream_connection</span><span class="pun">(</span><span class="pln">conn</span><span class="pun">)</span></code></li><li class="L9"><code class="language-python"><span class="pln">                conn</span><span class="pun">.</span><span class="pln">upstream_node</span><span class="pun">.</span><span class="pln">append_downstream_connection</span><span class="pun">(</span><span class="pln">conn</span><span class="pun">)</span></code></li><li class="L0"><code class="language-python"></code></li><li class="L1"><code class="language-python"></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> train</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> labels</span><span class="pun">,</span><span class="pln"> data_set</span><span class="pun">,</span><span class="pln"> rate</span><span class="pun">,</span><span class="pln"> iteration</span><span class="pun">):</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L4"><code class="language-python"><span class="str">        训练神经网络</span></code></li><li class="L5"><code class="language-python"><span class="str">        labels: 数组，训练样本标签。每个元素是一个样本的标签。</span></code></li><li class="L6"><code class="language-python"><span class="str">        data_set: 二维数组，训练样本特征。每个元素是一个样本的特征。</span></code></li><li class="L7"><code class="language-python"><span class="str">        '''</span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">iteration</span><span class="pun">):</span></code></li><li class="L9"><code class="language-python"><span class="pln">            </span><span class="kwd">for</span><span class="pln"> d </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">len</span><span class="pun">(</span><span class="pln">data_set</span><span class="pun">)):</span></code></li><li class="L0"><code class="language-python"><span class="pln">                self</span><span class="pun">.</span><span class="pln">train_one_sample</span><span class="pun">(</span><span class="pln">labels</span><span class="pun">[</span><span class="pln">d</span><span class="pun">],</span><span class="pln"> data_set</span><span class="pun">[</span><span class="pln">d</span><span class="pun">],</span><span class="pln"> rate</span><span class="pun">)</span></code></li><li class="L1"><code class="language-python"></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> train_one_sample</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> label</span><span class="pun">,</span><span class="pln"> sample</span><span class="pun">,</span><span class="pln"> rate</span><span class="pun">):</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L4"><code class="language-python"><span class="str">        内部函数，用一个样本训练网络</span></code></li><li class="L5"><code class="language-python"><span class="str">        '''</span></code></li><li class="L6"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">predict</span><span class="pun">(</span><span class="pln">sample</span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">calc_delta</span><span class="pun">(</span><span class="pln">label</span><span class="pun">)</span></code></li><li class="L8"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">update_weight</span><span class="pun">(</span><span class="pln">rate</span><span class="pun">)</span></code></li><li class="L9"><code class="language-python"></code></li><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_delta</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> label</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        内部函数，计算每个节点的delta</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        output_nodes </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[-</span><span class="lit">1</span><span class="pun">].</span><span class="pln">nodes</span></code></li><li class="L5"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">len</span><span class="pun">(</span><span class="pln">label</span><span class="pun">)):</span></code></li><li class="L6"><code class="language-python"><span class="pln">            output_nodes</span><span class="pun">[</span><span class="pln">i</span><span class="pun">].</span><span class="pln">calc_output_layer_delta</span><span class="pun">(</span><span class="pln">label</span><span class="pun">[</span><span class="pln">i</span><span class="pun">])</span></code></li><li class="L7"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> layer </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[-</span><span class="lit">2</span><span class="pun">::-</span><span class="lit">1</span><span class="pun">]:</span></code></li><li class="L8"><code class="language-python"><span class="pln">            </span><span class="kwd">for</span><span class="pln"> node </span><span class="kwd">in</span><span class="pln"> layer</span><span class="pun">.</span><span class="pln">nodes</span><span class="pun">:</span></code></li><li class="L9"><code class="language-python"><span class="pln">                node</span><span class="pun">.</span><span class="pln">calc_hidden_layer_delta</span><span class="pun">()</span></code></li><li class="L0"><code class="language-python"></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> update_weight</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> rate</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        内部函数，更新每个连接权重</span></code></li><li class="L4"><code class="language-python"><span class="str">        '''</span></code></li><li class="L5"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> layer </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[:-</span><span class="lit">1</span><span class="pun">]:</span></code></li><li class="L6"><code class="language-python"><span class="pln">            </span><span class="kwd">for</span><span class="pln"> node </span><span class="kwd">in</span><span class="pln"> layer</span><span class="pun">.</span><span class="pln">nodes</span><span class="pun">:</span></code></li><li class="L7"><code class="language-python"><span class="pln">                </span><span class="kwd">for</span><span class="pln"> conn </span><span class="kwd">in</span><span class="pln"> node</span><span class="pun">.</span><span class="pln">downstream</span><span class="pun">:</span></code></li><li class="L8"><code class="language-python"><span class="pln">                    conn</span><span class="pun">.</span><span class="pln">update_weight</span><span class="pun">(</span><span class="pln">rate</span><span class="pun">)</span></code></li><li class="L9"><code class="language-python"></code></li><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_gradient</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        内部函数，计算每个连接的梯度</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> layer </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[:-</span><span class="lit">1</span><span class="pun">]:</span></code></li><li class="L5"><code class="language-python"><span class="pln">            </span><span class="kwd">for</span><span class="pln"> node </span><span class="kwd">in</span><span class="pln"> layer</span><span class="pun">.</span><span class="pln">nodes</span><span class="pun">:</span></code></li><li class="L6"><code class="language-python"><span class="pln">                </span><span class="kwd">for</span><span class="pln"> conn </span><span class="kwd">in</span><span class="pln"> node</span><span class="pun">.</span><span class="pln">downstream</span><span class="pun">:</span></code></li><li class="L7"><code class="language-python"><span class="pln">                    conn</span><span class="pun">.</span><span class="pln">calc_gradient</span><span class="pun">()</span></code></li><li class="L8"><code class="language-python"></code></li><li class="L9"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> get_gradient</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> label</span><span class="pun">,</span><span class="pln"> sample</span><span class="pun">):</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L1"><code class="language-python"><span class="str">        获得网络在一个样本下，每个连接上的梯度</span></code></li><li class="L2"><code class="language-python"><span class="str">        label: 样本标签</span></code></li><li class="L3"><code class="language-python"><span class="str">        sample: 样本输入</span></code></li><li class="L4"><code class="language-python"><span class="str">        '''</span></code></li><li class="L5"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">predict</span><span class="pun">(</span><span class="pln">sample</span><span class="pun">)</span></code></li><li class="L6"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">calc_delta</span><span class="pun">(</span><span class="pln">label</span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">calc_gradient</span><span class="pun">()</span></code></li><li class="L8"><code class="language-python"></code></li><li class="L9"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> predict</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> sample</span><span class="pun">):</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L1"><code class="language-python"><span class="str">        根据输入的样本预测输出值</span></code></li><li class="L2"><code class="language-python"><span class="str">        sample: 数组，样本的特征，也就是网络的输入向量</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[</span><span class="lit">0</span><span class="pun">].</span><span class="pln">set_output</span><span class="pun">(</span><span class="pln">sample</span><span class="pun">)</span></code></li><li class="L5"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="lit">1</span><span class="pun">,</span><span class="pln"> len</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">)):</span></code></li><li class="L6"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[</span><span class="pln">i</span><span class="pun">].</span><span class="pln">calc_output</span><span class="pun">()</span></code></li><li class="L7"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> map</span><span class="pun">(</span><span class="kwd">lambda</span><span class="pln"> node</span><span class="pun">:</span><span class="pln"> node</span><span class="pun">.</span><span class="pln">output</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[-</span><span class="lit">1</span><span class="pun">].</span><span class="pln">nodes</span><span class="pun">[:-</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L8"><code class="language-python"></code></li><li class="L9"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> dump</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L1"><code class="language-python"><span class="str">        打印网络信息</span></code></li><li class="L2"><code class="language-python"><span class="str">        '''</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> layer </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">:</span></code></li><li class="L4"><code class="language-python"><span class="pln">            layer</span><span class="pun">.</span><span class="pln">dump</span><span class="pun">()</span></code></li></ol></pre><p data-anchor-id="zp3w">至此，实现了一个基本的全连接神经网络。可以看到，同神经网络的强大学习能力相比，其实现还算是很容易的。</p><div class="md-section-divider"></div><h3 data-anchor-id="kwer" id="梯度检查">梯度检查</h3><p data-anchor-id="umv1">怎么保证自己写的神经网络没有BUG呢？事实上这是一个非常重要的问题。一方面，千辛万苦想到一个算法，结果效果不理想，那么是算法本身错了还是代码实现错了呢？定位这种问题肯定要花费大量的时间和精力。另一方面，由于神经网络的复杂性，我们几乎无法事先知道神经网络的输入和输出，因此类似TDD(测试驱动开发)这样的开发方法似乎也不可行。</p><p data-anchor-id="3tcx">办法还是有滴，就是利用梯度检查来确认程序是否正确。梯度检查的思路如下：</p><p data-anchor-id="thsl">对于梯度下降算法：</p><div class="md-section-divider"></div><p data-anchor-id="4x72"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-154-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1411.7080693679925 8434.127487084877 2412.072891782453" style="width: 19.597ex; height: 5.565ex; vertical-align: -2.419ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g><use href="#MJMAIN-2190" x="1630" y="0"></use><g transform="translate(2908,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-2212" x="4483" y="0"></use><use href="#MJMATHI-3B7" x="5483" y="0"></use><g transform="translate(6274,0)"><rect stroke="none" width="2039" height="60" x="0" y="220"></rect><g transform="translate(131,676)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-154">
w_{ji}\gets w_{ji}-\eta\frac{\partial{E_d}}{\partial{w_{ji}}}
</script><p></p><p data-anchor-id="8rxv">来说，这里关键之处在于<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-155-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1044.710439052962 1717.635785162182 1686.4208781059244" style="width: 3.992ex; height: 3.871ex; vertical-align: -1.573ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="1477" height="60" x="0" y="220"></rect><g transform="translate(110,519)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-45"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-414)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-77"></use><g transform="translate(506,-107)"><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-155">\frac{\partial{E_d}}{\partial{w_{ji}}}</script>的计算一定要正确，而它是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-156-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -700.2000693679926 1208.6703999511576 877.1783133290371" style="width: 2.782ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-156">E_d</script>对<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-157-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-157">w_{ji}</script>的<em>偏导数</em>。而根据导数的定义：</p><div class="md-section-divider"></div><p data-anchor-id="004m"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-158-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1491.4079322712182 13794.724582616103 2278.9056027192864" style="width: 32.056ex; height: 5.323ex; vertical-align: -1.935ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-66"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2032" x="804" y="583"></use><use href="#MJMAIN-28" x="863" y="0"></use><use href="#MJMATHI-3B8" x="1252" y="0"></use><use href="#MJMAIN-29" x="1722" y="0"></use><use href="#MJMAIN-3D" x="2389" y="0"></use><g transform="translate(3445,0)"><g transform="translate(175,0)"><use href="#MJMAIN-6C"></use><use href="#MJMAIN-69" x="278" y="0"></use><use href="#MJMAIN-6D" x="557" y="0"></use></g><g transform="translate(0,-640)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-3F5"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="406" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-3E" x="1185" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="1963" y="0"></use></g></g><g transform="translate(5474,0)"><rect stroke="none" width="8199" height="60" x="0" y="220"></rect><g transform="translate(60,721)"><use href="#MJMATHI-66"></use><use href="#MJMAIN-28" x="550" y="0"></use><use href="#MJMATHI-3B8" x="940" y="0"></use><use href="#MJMAIN-2B" x="1631" y="0"></use><use href="#MJMATHI-3F5" x="2632" y="0"></use><use href="#MJMAIN-29" x="3038" y="0"></use><use href="#MJMAIN-2212" x="3650" y="0"></use><use href="#MJMATHI-66" x="4651" y="0"></use><use href="#MJMAIN-28" x="5201" y="0"></use><use href="#MJMATHI-3B8" x="5591" y="0"></use><use href="#MJMAIN-2212" x="6283" y="0"></use><use href="#MJMATHI-3F5" x="7283" y="0"></use><use href="#MJMAIN-29" x="7690" y="0"></use></g><g transform="translate(3646,-686)"><use href="#MJMAIN-32"></use><use href="#MJMATHI-3F5" x="500" y="0"></use></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-158">
f'(\theta)=\lim_{\epsilon->0}\frac{f(\theta+\epsilon)-f(\theta-\epsilon)}{2\epsilon}
</script><p></p><p data-anchor-id="cztu">对于任意<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-159-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -725.2000693679926 469.5 755.4001387359851" style="width: 1.089ex; height: 1.815ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B8"></use></g></svg></span><script type="math/tex" id="MathJax-Element-159">\theta</script>的导数值，我们都可以用等式右边来近似计算。我们把<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-160-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -700.2000693679926 1208.6703999511576 877.1783133290371" style="width: 2.782ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use></g></svg></span><script type="math/tex" id="MathJax-Element-160">E_d</script>看做是<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-161-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-161">w_{ji}</script>的函数，即<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-162-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 3340.157340090561 1084.3570288792273" style="width: 7.742ex; height: 2.54ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use><use href="#MJMAIN-28" x="1208" y="0"></use><g transform="translate(1598,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-29" x="2950" y="0"></use></g></svg></span><script type="math/tex" id="MathJax-Element-162">E_d(w_{ji})</script>，那么根据导数定义，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-163-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1166.4591763944914 3123.131003731423 1808.1696154474537" style="width: 7.258ex; height: 4.234ex; vertical-align: -1.573ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="2883" height="60" x="0" y="220"></rect><g transform="translate(60,616)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-45"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-64" x="1044" y="-213"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-28" x="1208" y="0"></use><g transform="translate(1130,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-77"></use><g transform="translate(506,-107)"><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="2950" y="0"></use></g></g><g transform="translate(762,-414)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-77"></use><g transform="translate(506,-107)"><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-163">\frac{\partial{E_d(w_{ji})}}{\partial{w_{ji}}}</script>应该等于：</p><div class="md-section-divider"></div><p data-anchor-id="fy0k"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-164-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1535.3648224144604 17716.497884768578 2535.7296448289208" style="width: 41.129ex; height: 5.927ex; vertical-align: -2.419ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="4027" height="60" x="0" y="220"></rect><g transform="translate(60,765)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use><use href="#MJMAIN-28" x="1208" y="0"></use><g transform="translate(1598,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-29" x="2950" y="0"></use></g></g><g transform="translate(1053,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g><use href="#MJMAIN-3D" x="4545" y="0"></use><g transform="translate(5601,0)"><g transform="translate(175,0)"><use href="#MJMAIN-6C"></use><use href="#MJMAIN-69" x="278" y="0"></use><use href="#MJMAIN-6D" x="557" y="0"></use></g><g transform="translate(0,-640)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-3F5"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="406" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-3E" x="1185" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-30" x="1963" y="0"></use></g></g><g transform="translate(7630,0)"><rect stroke="none" width="9965" height="60" x="0" y="220"></rect><g transform="translate(60,765)"><use href="#MJMATHI-66"></use><use href="#MJMAIN-28" x="550" y="0"></use><g transform="translate(940,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-2B" x="2514" y="0"></use><use href="#MJMATHI-3F5" x="3515" y="0"></use><use href="#MJMAIN-29" x="3921" y="0"></use><use href="#MJMAIN-2212" x="4533" y="0"></use><use href="#MJMATHI-66" x="5534" y="0"></use><use href="#MJMAIN-28" x="6084" y="0"></use><g transform="translate(6474,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-2212" x="8049" y="0"></use><use href="#MJMATHI-3F5" x="9049" y="0"></use><use href="#MJMAIN-29" x="9456" y="0"></use></g><g transform="translate(4529,-686)"><use href="#MJMAIN-32"></use><use href="#MJMATHI-3F5" x="500" y="0"></use></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-164">
\frac{\partial{E_d(w_{ji})}}{\partial{w_{ji}}}=\lim_{\epsilon->0}\frac{f(w_{ji}+\epsilon)-f(w_{ji}-\epsilon)}{2\epsilon}
</script><p></p><p data-anchor-id="2mo8">如果把<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-165-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -451.20006936799257 406.5 482.40013873598514" style="width: 0.968ex; height: 1.089ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3F5"></use></g></svg></span><script type="math/tex" id="MathJax-Element-165">\epsilon</script>设置为一个很小的数（比如<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-166-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -892.5988819420962 2005.3895731375944 934.7989513100888" style="width: 4.718ex; height: 2.177ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-31"></use><use href="#MJMAIN-30" x="500" y="0"></use><g transform="translate(1001,393)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="778" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-166">10^{-4}</script>），那么上式可以写成：</p><div class="md-section-divider"></div><p data-anchor-id="9cc8"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-167-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1535.3648224144604 20087.01709850557 2535.7296448289208" style="width: 46.694ex; height: 5.927ex; vertical-align: -2.419ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="4027" height="60" x="0" y="220"></rect><g transform="translate(60,765)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-45"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-64" x="1044" y="-213"></use><use href="#MJMAIN-28" x="1208" y="0"></use><g transform="translate(1598,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-29" x="2950" y="0"></use></g></g><g transform="translate(1053,-687)"><use href="#MJMAIN-2202"></use><g transform="translate(567,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g><use href="#MJMAIN-2248" x="4545" y="0"></use><g transform="translate(5721,0)"><rect stroke="none" width="9965" height="60" x="0" y="220"></rect><g transform="translate(60,765)"><use href="#MJMATHI-66"></use><use href="#MJMAIN-28" x="550" y="0"></use><g transform="translate(940,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-2B" x="2514" y="0"></use><use href="#MJMATHI-3F5" x="3515" y="0"></use><use href="#MJMAIN-29" x="3921" y="0"></use><use href="#MJMAIN-2212" x="4533" y="0"></use><use href="#MJMATHI-66" x="5534" y="0"></use><use href="#MJMAIN-28" x="6084" y="0"></use><g transform="translate(6474,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-2212" x="8049" y="0"></use><use href="#MJMATHI-3F5" x="9049" y="0"></use><use href="#MJMAIN-29" x="9456" y="0"></use></g><g transform="translate(4529,-686)"><use href="#MJMAIN-32"></use><use href="#MJMATHI-3F5" x="500" y="0"></use></g></g><use href="#MJMAIN-28" x="17974" y="0"></use><g transform="translate(18363,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-36" x="19197" y="0"></use><use href="#MJMAIN-29" x="19697" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-167">
\frac{\partial{E_d(w_{ji})}}{\partial{w_{ji}}}\approx\frac{f(w_{ji}+\epsilon)-f(w_{ji}-\epsilon)}{2\epsilon}\qquad(式6)
</script><p></p><p data-anchor-id="e7zn">我们就可以利用式6，来计算梯度<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-168-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1044.710439052962 1717.635785162182 1686.4208781059244" style="width: 3.992ex; height: 3.871ex; vertical-align: -1.573ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(120,0)"><rect stroke="none" width="1477" height="60" x="0" y="220"></rect><g transform="translate(110,519)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-45"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-64" x="1044" y="-213"></use></g></g><g transform="translate(60,-414)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2202"></use><g transform="translate(401,0)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-77"></use><g transform="translate(506,-107)"><use transform="scale(0.5000000000000001)" href="#MJMATHI-6A"></use><use transform="scale(0.5000000000000001)" href="#MJMATHI-69" x="412" y="0"></use></g></g></g></g></g></svg></span><script type="math/tex" id="MathJax-Element-168">\frac{\partial{E_d}}{\partial{w_{ji}}}</script>的值，然后同我们神经网络代码中计算出来的梯度值进行比较。如果两者的差别<strong>非常的小</strong>，那么就说明我们的代码是正确的。</p><p data-anchor-id="9vkf">下面是梯度检查的代码。如果我们想检查参数<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-169-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-169">w_{ji}</script>的梯度是否正确，我们需要以下几个步骤：</p><ol data-anchor-id="naur">
<li>首先使用一个样本<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-170-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 523.5 744.4001387359851" style="width: 1.21ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-64"></use></g></svg></span><script type="math/tex" id="MathJax-Element-170">d</script>对神经网络进行训练，这样就能获得每个权重的梯度。</li>
<li>将<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-171-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-171">w_{ji}</script>加上一个很小的值(<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-172-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -892.5988819420962 2005.3895731375944 934.7989513100888" style="width: 4.718ex; height: 2.177ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-31"></use><use href="#MJMAIN-30" x="500" y="0"></use><g transform="translate(1001,393)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="778" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-172">10^{-4}</script>)，重新计算神经网络在这个样本<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-173-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 523.5 744.4001387359851" style="width: 1.21ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-64"></use></g></svg></span><script type="math/tex" id="MathJax-Element-173">d</script>下的<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-174-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -700.2000693679926 1759.153029104885 928.0900015744686" style="width: 4.113ex; height: 2.177ex; vertical-align: -0.605ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><g transform="translate(738,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-64"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2B" x="523" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-174">E_{d+}</script>。</li>
<li>将<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-175-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-175">w_{ji}</script>减上一个很小的值(<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-176-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -892.5988819420962 2005.3895731375944 934.7989513100888" style="width: 4.718ex; height: 2.177ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-31"></use><use href="#MJMAIN-30" x="500" y="0"></use><g transform="translate(1001,393)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-34" x="778" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-176">10^{-4}</script>)，重新计算神经网络在这个样本<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-177-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -714.2000693679926 523.5 744.4001387359851" style="width: 1.21ex; height: 1.694ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-64"></use></g></svg></span><script type="math/tex" id="MathJax-Element-177">d</script>下的<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-178-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -700.2000693679926 1759.153029104885 877.1783133290371" style="width: 4.113ex; height: 2.056ex; vertical-align: -0.484ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-45"></use><g transform="translate(738,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-64"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2212" x="523" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-178">E_{d-}</script>。</li>
<li>根据式6计算出期望的梯度值，和第一步获得的梯度值进行比较，它们应该几乎想等(至少4位有效数字相同)。</li>
</ol><p data-anchor-id="wcu1">当然，我们可以重复上面的过程，对每个权重<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-179-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 1352.486940139403 777.3570288792274" style="width: 3.145ex; height: 1.815ex; vertical-align: -0.847ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-179">w_{ji}</script>都进行检查。也可以使用多个样本重复检查。</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="3bdm" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">def</span><span class="pln"> gradient_check</span><span class="pun">(</span><span class="pln">network</span><span class="pun">,</span><span class="pln"> sample_feature</span><span class="pun">,</span><span class="pln"> sample_label</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">    梯度检查</span></code></li><li class="L3"><code class="language-python"><span class="str">    network: 神经网络对象</span></code></li><li class="L4"><code class="language-python"><span class="str">    sample_feature: 样本的特征</span></code></li><li class="L5"><code class="language-python"><span class="str">    sample_label: 样本的标签</span></code></li><li class="L6"><code class="language-python"><span class="str">    '''</span></code></li><li class="L7"><code class="language-python"><span class="pln">    </span><span class="com"># 计算网络误差</span></code></li><li class="L8"><code class="language-python"><span class="pln">    network_error </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">lambda</span><span class="pln"> vec1</span><span class="pun">,</span><span class="pln"> vec2</span><span class="pun">:</span><span class="pln"> \</span></code></li><li class="L9"><code class="language-python"><span class="pln">            </span><span class="lit">0.5</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> reduce</span><span class="pun">(</span><span class="kwd">lambda</span><span class="pln"> a</span><span class="pun">,</span><span class="pln"> b</span><span class="pun">:</span><span class="pln"> a </span><span class="pun">+</span><span class="pln"> b</span><span class="pun">,</span><span class="pln"> </span></code></li><li class="L0"><code class="language-python"><span class="pln">                      map</span><span class="pun">(</span><span class="kwd">lambda</span><span class="pln"> v</span><span class="pun">:</span><span class="pln"> </span><span class="pun">(</span><span class="pln">v</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span><span class="pln"> </span><span class="pun">-</span><span class="pln"> v</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> </span><span class="pun">(</span><span class="pln">v</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span><span class="pln"> </span><span class="pun">-</span><span class="pln"> v</span><span class="pun">[</span><span class="lit">1</span><span class="pun">]),</span></code></li><li class="L1"><code class="language-python"><span class="pln">                          zip</span><span class="pun">(</span><span class="pln">vec1</span><span class="pun">,</span><span class="pln"> vec2</span><span class="pun">)))</span></code></li><li class="L2"><code class="language-python"></code></li><li class="L3"><code class="language-python"><span class="pln">    </span><span class="com"># 获取网络在当前样本下每个连接的梯度</span></code></li><li class="L4"><code class="language-python"><span class="pln">    network</span><span class="pun">.</span><span class="pln">get_gradient</span><span class="pun">(</span><span class="pln">sample_feature</span><span class="pun">,</span><span class="pln"> sample_label</span><span class="pun">)</span></code></li><li class="L5"><code class="language-python"></code></li><li class="L6"><code class="language-python"><span class="pln">    </span><span class="com"># 对每个权重做梯度检查    </span></code></li><li class="L7"><code class="language-python"><span class="pln">    </span><span class="kwd">for</span><span class="pln"> conn </span><span class="kwd">in</span><span class="pln"> network</span><span class="pun">.</span><span class="pln">connections</span><span class="pun">.</span><span class="pln">connections</span><span class="pun">:</span><span class="pln"> </span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="com"># 获取指定连接的梯度</span></code></li><li class="L9"><code class="language-python"><span class="pln">        actual_gradient </span><span class="pun">=</span><span class="pln"> conn</span><span class="pun">.</span><span class="pln">get_gradient</span><span class="pun">()</span></code></li><li class="L0"><code class="language-python"></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="com"># 增加一个很小的值，计算网络的误差</span></code></li><li class="L2"><code class="language-python"><span class="pln">        epsilon </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0.0001</span></code></li><li class="L3"><code class="language-python"><span class="pln">        conn</span><span class="pun">.</span><span class="pln">weight </span><span class="pun">+=</span><span class="pln"> epsilon</span></code></li><li class="L4"><code class="language-python"><span class="pln">        error1 </span><span class="pun">=</span><span class="pln"> network_error</span><span class="pun">(</span><span class="pln">network</span><span class="pun">.</span><span class="pln">predict</span><span class="pun">(</span><span class="pln">sample_feature</span><span class="pun">),</span><span class="pln"> sample_label</span><span class="pun">)</span></code></li><li class="L5"><code class="language-python"></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="com"># 减去一个很小的值，计算网络的误差</span></code></li><li class="L7"><code class="language-python"><span class="pln">        conn</span><span class="pun">.</span><span class="pln">weight </span><span class="pun">-=</span><span class="pln"> </span><span class="lit">2</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> epsilon </span><span class="com"># 刚才加过了一次，因此这里需要减去2倍</span></code></li><li class="L8"><code class="language-python"><span class="pln">        error2 </span><span class="pun">=</span><span class="pln"> network_error</span><span class="pun">(</span><span class="pln">network</span><span class="pun">.</span><span class="pln">predict</span><span class="pun">(</span><span class="pln">sample_feature</span><span class="pun">),</span><span class="pln"> sample_label</span><span class="pun">)</span></code></li><li class="L9"><code class="language-python"></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="com"># 根据式6计算期望的梯度值</span></code></li><li class="L1"><code class="language-python"><span class="pln">        expected_gradient </span><span class="pun">=</span><span class="pln"> </span><span class="pun">(</span><span class="pln">error2 </span><span class="pun">-</span><span class="pln"> error1</span><span class="pun">)</span><span class="pln"> </span><span class="pun">/</span><span class="pln"> </span><span class="pun">(</span><span class="lit">2</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> epsilon</span><span class="pun">)</span></code></li><li class="L2"><code class="language-python"></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="com"># 打印</span></code></li><li class="L4"><code class="language-python"><span class="pln">        </span><span class="kwd">print</span><span class="pln"> </span><span class="str">'expected gradient: \t%f\nactual gradient: \t%f'</span><span class="pln"> </span><span class="pun">%</span><span class="pln"> </span><span class="pun">(</span></code></li><li class="L5"><code class="language-python"><span class="pln">            expected_gradient</span><span class="pun">,</span><span class="pln"> actual_gradient</span><span class="pun">)</span></code></li></ol></pre><p data-anchor-id="5tf6">至此，会推导、会实现、会抓BUG，你已经摸到深度学习的大门了。接下来还需要不断的实践，我们用刚刚写过的神经网络去识别手写数字。</p><div class="md-section-divider"></div><h2 data-anchor-id="pvb0" id="神经网络实战手写数字识别">神经网络实战——手写数字识别</h2><p data-anchor-id="f94n">针对这个任务，我们采用业界非常流行的MNIST数据集。MNIST大约有60000个手写字母的训练样本，我们使用它训练我们的神经网络，然后再用训练好的网络去识别手写数字。</p><p data-anchor-id="6acg">手写数字识别是个比较简单的任务，数字只可能是0-9中的一个，这是个10分类问题。</p><div class="md-section-divider"></div><h3 data-anchor-id="5e7q" id="超参数的确定">超参数的确定</h3><p data-anchor-id="394n">我们首先需要确定网络的层数和每层的节点数。关于第一个问题，实际上并没有什么理论化的方法，大家都是根据经验来拍，如果没有经验的话就随便拍一个。然后，你可以多试几个值，训练不同层数的神经网络，看看哪个效果最好就用哪个。嗯，现在你可能明白为什么说深度学习是个手艺活了，有些手艺很让人无语，而有些手艺还是很有技术含量的。</p><p data-anchor-id="vwro">不过，有些基本道理我们还是明白的，我们知道网络层数越多越好，也知道层数越多训练难度越大。对于全连接网络，隐藏层最好不要超过三层。那么，我们可以先试试仅有一个隐藏层的神经网络效果怎么样。毕竟模型小的话，训练起来也快些(刚开始玩模型的时候，都希望快点看到结果)。</p><p data-anchor-id="0pr1">输入层节点数是确定的。因为MNIST数据集每个训练数据是28*28的图片，共784个像素，因此，输入层节点数应该是784，每个像素对应一个输入节点。</p><p data-anchor-id="zx0q">输出层节点数也是确定的。因为是10分类，我们可以用10个节点，每个节点对应一个分类。输出层10个节点中，输出最大值的那个节点对应的分类，就是模型的预测结果。</p><p data-anchor-id="nv8r">隐藏层节点数量是不好确定的，从1到100万都可以。下面有几个经验公式：</p><div class="md-section-divider"></div><p data-anchor-id="h3b2"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-180-Frame" style="font-size: 100%; display: inline-block;"><span style="display: inline-block; white-space: nowrap; padding: 1px 0px;"><span style="display: inline-block; position: relative; width: 102.823ex; height: 23.589ex; vertical-align: -11.25ex;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -5352.350662109928 44270.673387096766 10204.701324219857" style="width: 102.823ex; height: 23.71ex; position: absolute; bottom: 0ex; left: 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(18182,0)"><g transform="translate(263,0)"><g transform="translate(0,4349)"><use href="#MJMATHI-6D"></use><use href="#MJMAIN-3D" x="1156" y="0"></use><g transform="translate(2212,0)"><use href="#MJMAIN-221A" x="0" y="122"></use><rect stroke="none" width="2121" height="60" x="833" y="863"></rect><g transform="translate(833,0)"><use href="#MJMATHI-6E"></use><use href="#MJMAIN-2B" x="822" y="0"></use><use href="#MJMATHI-6C" x="1823" y="0"></use></g></g><use href="#MJMAIN-2B" x="5390" y="0"></use><use href="#MJMATHI-3B1" x="6390" y="0"></use></g><g transform="translate(0,3047)"><use href="#MJMATHI-6D"></use><use href="#MJMAIN-3D" x="1156" y="0"></use><use href="#MJMATHI-6C" x="2212" y="0"></use><use href="#MJMATHI-6F" x="2511" y="0"></use><g transform="translate(2996,0)"><use href="#MJMATHI-67"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-32" x="675" y="-213"></use></g><use href="#MJMATHI-6E" x="3927" y="0"></use></g><g transform="translate(0,1617)"><use href="#MJMATHI-6D"></use><use href="#MJMAIN-3D" x="1156" y="0"></use><g transform="translate(2212,0)"><use href="#MJMAIN-221A" x="0" y="17"></use><rect stroke="none" width="899" height="60" x="833" y="758"></rect><g transform="translate(833,0)"><use href="#MJMATHI-6E"></use><use href="#MJMATHI-6C" x="600" y="0"></use></g></g></g><g transform="translate(0,181)"><use href="#MJMATHI-6D"></use><use href="#MJMAIN-3A" x="1156" y="0"></use><g transform="translate(1712,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">隐</text></g><g transform="translate(2545,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">藏</text></g><g transform="translate(3379,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">层</text></g><g transform="translate(4212,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">节</text></g><g transform="translate(5045,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">点</text></g><g transform="translate(5879,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">数</text></g></g><g transform="translate(0,-1369)"><use href="#MJMATHI-6E"></use><use href="#MJMAIN-3A" x="878" y="0"></use><g transform="translate(1434,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">输</text></g><g transform="translate(2267,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">入</text></g><g transform="translate(3101,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">层</text></g><g transform="translate(3934,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">节</text></g><g transform="translate(4767,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">点</text></g><g transform="translate(5601,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">数</text></g></g><g transform="translate(0,-2919)"><use href="#MJMATHI-6C"></use><use href="#MJMAIN-3A" x="576" y="0"></use><g transform="translate(1132,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">输</text></g><g transform="translate(1965,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">出</text></g><g transform="translate(2799,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">层</text></g><g transform="translate(3632,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">节</text></g><g transform="translate(4465,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">点</text></g><g transform="translate(5299,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">数</text></g></g><g transform="translate(0,-4469)"><use href="#MJMATHI-3B1"></use><use href="#MJMAIN-3A" x="918" y="0"></use><use href="#MJMAIN-31" x="1474" y="0"></use><g transform="translate(1975,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">到</text></g><g transform="translate(2808,0)"><use href="#MJMAIN-31"></use><use href="#MJMAIN-30" x="500" y="0"></use></g><g transform="translate(3809,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">之</text></g><g transform="translate(4642,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">间</text></g><g transform="translate(5476,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">的</text></g><g transform="translate(6309,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">常</text></g><g transform="translate(7142,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">数</text></g></g></g></g><g transform="translate(41690,0)"><g transform="translate(0,4349)"><g id="mjx-eqn-41"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-31" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,3047)"><g id="mjx-eqn-42"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-32" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,1617)"><g id="mjx-eqn-43"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-33" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,181)"><g id="mjx-eqn-44"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-34" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-1369)"><g id="mjx-eqn-45"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-35" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-2919)"><g id="mjx-eqn-46"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-36" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g><g transform="translate(0,-4469)"><g id="mjx-eqn-47"><use href="#MJMAIN-28"></use><use href="#MJMAIN-34" x="389" y="0"></use><use href="#MJMAIN-37" x="890" y="0"></use><use href="#MJMAIN-29" x="1390" y="0"></use></g></g></g></g></svg></span></span></span></div><script type="math/tex; mode=display" id="MathJax-Element-180">
\begin{align}
&m=\sqrt{n+l}+\alpha\\
&m=log_2n\\
&m=\sqrt{nl}\\
&m:隐藏层节点数\\
&n:输入层节点数\\
&l:输出层节点数\\
&\alpha:1到10之间的常数
\end{align}
</script><p></p><p data-anchor-id="a5gs">因此，我们可以先根据上面的公式设置一个隐藏层节点数。如果有时间，我们可以设置不同的节点数，分别训练，看看哪个效果最好就用哪个。我们先拍一个，设隐藏层节点数为300吧。</p><p data-anchor-id="levm">对于3层<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-181-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -697.2000693679926 5893.88888888889 739.4001387359851" style="width: 13.669ex; height: 1.694ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-37"></use><use href="#MJMAIN-38" x="500" y="0"></use><use href="#MJMAIN-34" x="1001" y="0"></use><use href="#MJMAIN-2217" x="1723" y="0"></use><g transform="translate(2446,0)"><use href="#MJMAIN-33"></use><use href="#MJMAIN-30" x="500" y="0"></use><use href="#MJMAIN-30" x="1001" y="0"></use></g><use href="#MJMAIN-2217" x="4170" y="0"></use><g transform="translate(4892,0)"><use href="#MJMAIN-31"></use><use href="#MJMAIN-30" x="500" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-181">784*300*10</script>的全连接网络，总共有<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-182-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -770.2000693679926 17960.27777777778 1040.4001387359851" style="width: 41.734ex; height: 2.419ex; vertical-align: -0.726ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMAIN-33"></use><use href="#MJMAIN-30" x="500" y="0"></use><use href="#MJMAIN-30" x="1001" y="0"></use><use href="#MJMAIN-2217" x="1723" y="0"></use><use href="#MJMAIN-28" x="2446" y="0"></use><g transform="translate(2835,0)"><use href="#MJMAIN-37"></use><use href="#MJMAIN-38" x="500" y="0"></use><use href="#MJMAIN-34" x="1001" y="0"></use></g><use href="#MJMAIN-2B" x="4559" y="0"></use><use href="#MJMAIN-31" x="5560" y="0"></use><use href="#MJMAIN-29" x="6060" y="0"></use><use href="#MJMAIN-2B" x="6672" y="0"></use><g transform="translate(7673,0)"><use href="#MJMAIN-31"></use><use href="#MJMAIN-30" x="500" y="0"></use></g><use href="#MJMAIN-2217" x="8896" y="0"></use><use href="#MJMAIN-28" x="9619" y="0"></use><g transform="translate(10008,0)"><use href="#MJMAIN-33"></use><use href="#MJMAIN-30" x="500" y="0"></use><use href="#MJMAIN-30" x="1001" y="0"></use></g><use href="#MJMAIN-2B" x="11732" y="0"></use><use href="#MJMAIN-31" x="12733" y="0"></use><use href="#MJMAIN-29" x="13233" y="0"></use><use href="#MJMAIN-3D" x="13901" y="0"></use><g transform="translate(14957,0)"><use href="#MJMAIN-32"></use><use href="#MJMAIN-33" x="500" y="0"></use><use href="#MJMAIN-38" x="1001" y="0"></use><use href="#MJMAIN-35" x="1501" y="0"></use><use href="#MJMAIN-31" x="2002" y="0"></use><use href="#MJMAIN-30" x="2502" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-182">300*(784+1)+10*(300+1)=238510</script>个参数！神经网络之所以强大，是它提供了一种非常简单的方法去实现大量的参数。目前百亿参数、千亿样本的超大规模神经网络也是有的。因为MNIST只有6万个训练样本，参数太多了很容易过拟合，效果反而不好。</p><div class="md-section-divider"></div><h3 data-anchor-id="t39p" id="模型的训练和评估">模型的训练和评估</h3><p data-anchor-id="fist">MNIST数据集包含10000个测试样本。我们先用60000个训练样本训练我们的网络，然后再用测试样本对网络进行测试，计算识别错误率：</p><div class="md-section-divider"></div><p data-anchor-id="l63w"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-183-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1739.403416142186 10027.358781362007 2978.806832284372" style="width: 23.347ex; height: 6.895ex; vertical-align: -3.024ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">错</text><g transform="translate(833,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">误</text></g><g transform="translate(1666,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">率</text></g><use href="#MJMAIN-3D" x="2777" y="0"></use><g transform="translate(3954,0)"><rect stroke="none" width="5953" height="60" x="0" y="220"></rect><g transform="translate(60,834)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">错</text><g transform="translate(833,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">误</text></g><g transform="translate(1666,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">预</text></g><g transform="translate(2499,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">测</text></g><g transform="translate(3333,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">样</text></g><g transform="translate(4166,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">本</text></g><g transform="translate(4999,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">数</text></g></g><g transform="translate(1309,-856)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">总</text><g transform="translate(833,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">样</text></g><g transform="translate(1666,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">本</text></g><g transform="translate(2499,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">数</text></g></g></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-183">
错误率=\frac{错误预测样本数}{总样本数}
</script><p></p><p data-anchor-id="3xym">我们每训练10轮，评估一次准确率。当准确率开始下降时（出现了过拟合）终止训练。</p><div class="md-section-divider"></div><h3 data-anchor-id="zmnu" id="代码实现">代码实现</h3><p data-anchor-id="61xm">首先，我们需要把MNIST数据集处理为神经网络能够接受的形式。MNIST训练集的文件格式可以参考官方网站，这里不在赘述。每个训练样本是一个28*28的图像，我们按照行优先，把它转化为一个784维的向量。每个标签是0-9的值，我们将其转换为一个10维的one-hot向量：如果标签值为<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-184-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 600.5 494.40013873598514" style="width: 1.452ex; height: 1.089ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use></g></svg></span><script type="math/tex" id="MathJax-Element-184">n</script>，我们就把向量的第<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-185-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 600.5 494.40013873598514" style="width: 1.452ex; height: 1.089ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use></g></svg></span><script type="math/tex" id="MathJax-Element-185">n</script>维（从0开始编号）设置为0.9，而其它维设置为0.1。例如，向量[0.1,0.1,0.9,0.1,0.1,0.1,0.1,0.1,0.1,0.1]表示值2。</p><p data-anchor-id="g8cr">下面是处理MNIST数据的代码：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="2au6" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="com">#!/usr/bin/env python</span></code></li><li class="L1"><code class="language-python"><span class="com"># -*- coding: UTF-8 -*-</span></code></li><li class="L2"><code class="language-python"></code></li><li class="L3"><code class="language-python"><span class="kwd">import</span><span class="pln"> struct</span></code></li><li class="L4"><code class="language-python"><span class="kwd">from</span><span class="pln"> bp </span><span class="kwd">import</span><span class="pln"> </span><span class="pun">*</span></code></li><li class="L5"><code class="language-python"><span class="kwd">from</span><span class="pln"> datetime </span><span class="kwd">import</span><span class="pln"> datetime</span></code></li><li class="L6"><code class="language-python"></code></li><li class="L7"><code class="language-python"></code></li><li class="L8"><code class="language-python"><span class="com"># 数据加载器基类</span></code></li><li class="L9"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">Loader</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> path</span><span class="pun">,</span><span class="pln"> count</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        初始化加载器</span></code></li><li class="L3"><code class="language-python"><span class="str">        path: 数据文件路径</span></code></li><li class="L4"><code class="language-python"><span class="str">        count: 文件中的样本个数</span></code></li><li class="L5"><code class="language-python"><span class="str">        '''</span></code></li><li class="L6"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">path </span><span class="pun">=</span><span class="pln"> path</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">count </span><span class="pun">=</span><span class="pln"> count</span></code></li><li class="L8"><code class="language-python"></code></li><li class="L9"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> get_file_content</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L1"><code class="language-python"><span class="str">        读取文件内容</span></code></li><li class="L2"><code class="language-python"><span class="str">        '''</span></code></li><li class="L3"><code class="language-python"><span class="pln">        f </span><span class="pun">=</span><span class="pln"> open</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">path</span><span class="pun">,</span><span class="pln"> </span><span class="str">'rb'</span><span class="pun">)</span></code></li><li class="L4"><code class="language-python"><span class="pln">        content </span><span class="pun">=</span><span class="pln"> f</span><span class="pun">.</span><span class="pln">read</span><span class="pun">()</span></code></li><li class="L5"><code class="language-python"><span class="pln">        f</span><span class="pun">.</span><span class="pln">close</span><span class="pun">()</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> content</span></code></li><li class="L7"><code class="language-python"></code></li><li class="L8"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> to_int</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> byte</span><span class="pun">):</span></code></li><li class="L9"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L0"><code class="language-python"><span class="str">        将unsigned byte字符转换为整数</span></code></li><li class="L1"><code class="language-python"><span class="str">        '''</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> struct</span><span class="pun">.</span><span class="pln">unpack</span><span class="pun">(</span><span class="str">'B'</span><span class="pun">,</span><span class="pln"> byte</span><span class="pun">)[</span><span class="lit">0</span><span class="pun">]</span></code></li><li class="L3"><code class="language-python"></code></li><li class="L4"><code class="language-python"></code></li><li class="L5"><code class="language-python"><span class="com"># 图像数据加载器</span></code></li><li class="L6"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">ImageLoader</span><span class="pun">(</span><span class="typ">Loader</span><span class="pun">):</span></code></li><li class="L7"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> get_picture</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> content</span><span class="pun">,</span><span class="pln"> index</span><span class="pun">):</span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L9"><code class="language-python"><span class="str">        内部函数，从文件中获取图像</span></code></li><li class="L0"><code class="language-python"><span class="str">        '''</span></code></li><li class="L1"><code class="language-python"><span class="pln">        start </span><span class="pun">=</span><span class="pln"> index </span><span class="pun">*</span><span class="pln"> </span><span class="lit">28</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> </span><span class="lit">28</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> </span><span class="lit">16</span></code></li><li class="L2"><code class="language-python"><span class="pln">        picture </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="lit">28</span><span class="pun">):</span></code></li><li class="L4"><code class="language-python"><span class="pln">            picture</span><span class="pun">.</span><span class="pln">append</span><span class="pun">([])</span></code></li><li class="L5"><code class="language-python"><span class="pln">            </span><span class="kwd">for</span><span class="pln"> j </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="lit">28</span><span class="pun">):</span></code></li><li class="L6"><code class="language-python"><span class="pln">                picture</span><span class="pun">[</span><span class="pln">i</span><span class="pun">].</span><span class="pln">append</span><span class="pun">(</span></code></li><li class="L7"><code class="language-python"><span class="pln">                    self</span><span class="pun">.</span><span class="pln">to_int</span><span class="pun">(</span><span class="pln">content</span><span class="pun">[</span><span class="pln">start </span><span class="pun">+</span><span class="pln"> i </span><span class="pun">*</span><span class="pln"> </span><span class="lit">28</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> j</span><span class="pun">]))</span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> picture</span></code></li><li class="L9"><code class="language-python"></code></li><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> get_one_sample</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> picture</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        内部函数，将图像转化为样本的输入向量</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        sample </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L5"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="lit">28</span><span class="pun">):</span></code></li><li class="L6"><code class="language-python"><span class="pln">            </span><span class="kwd">for</span><span class="pln"> j </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="lit">28</span><span class="pun">):</span></code></li><li class="L7"><code class="language-python"><span class="pln">                sample</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">picture</span><span class="pun">[</span><span class="pln">i</span><span class="pun">][</span><span class="pln">j</span><span class="pun">])</span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> sample</span></code></li><li class="L9"><code class="language-python"></code></li><li class="L0"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> load</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L2"><code class="language-python"><span class="str">        加载数据文件，获得全部样本的输入向量</span></code></li><li class="L3"><code class="language-python"><span class="str">        '''</span></code></li><li class="L4"><code class="language-python"><span class="pln">        content </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">get_file_content</span><span class="pun">()</span></code></li><li class="L5"><code class="language-python"><span class="pln">        data_set </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> index </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">count</span><span class="pun">):</span></code></li><li class="L7"><code class="language-python"><span class="pln">            data_set</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span></code></li><li class="L8"><code class="language-python"><span class="pln">                self</span><span class="pun">.</span><span class="pln">get_one_sample</span><span class="pun">(</span></code></li><li class="L9"><code class="language-python"><span class="pln">                    self</span><span class="pun">.</span><span class="pln">get_picture</span><span class="pun">(</span><span class="pln">content</span><span class="pun">,</span><span class="pln"> index</span><span class="pun">)))</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> data_set</span></code></li><li class="L1"><code class="language-python"></code></li><li class="L2"><code class="language-python"></code></li><li class="L3"><code class="language-python"><span class="com"># 标签数据加载器</span></code></li><li class="L4"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">LabelLoader</span><span class="pun">(</span><span class="typ">Loader</span><span class="pun">):</span></code></li><li class="L5"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> load</span><span class="pun">(</span><span class="pln">self</span><span class="pun">):</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L7"><code class="language-python"><span class="str">        加载数据文件，获得全部样本的标签向量</span></code></li><li class="L8"><code class="language-python"><span class="str">        '''</span></code></li><li class="L9"><code class="language-python"><span class="pln">        content </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">get_file_content</span><span class="pun">()</span></code></li><li class="L0"><code class="language-python"><span class="pln">        labels </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> index </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">count</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">            labels</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">norm</span><span class="pun">(</span><span class="pln">content</span><span class="pun">[</span><span class="pln">index </span><span class="pun">+</span><span class="pln"> </span><span class="lit">8</span><span class="pun">]))</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> labels</span></code></li><li class="L4"><code class="language-python"></code></li><li class="L5"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> norm</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> label</span><span class="pun">):</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L7"><code class="language-python"><span class="str">        内部函数，将一个值转换为10维标签向量</span></code></li><li class="L8"><code class="language-python"><span class="str">        '''</span></code></li><li class="L9"><code class="language-python"><span class="pln">        label_vec </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L0"><code class="language-python"><span class="pln">        label_value </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">to_int</span><span class="pun">(</span><span class="pln">label</span><span class="pun">)</span></code></li><li class="L1"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="lit">10</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">            </span><span class="kwd">if</span><span class="pln"> i </span><span class="pun">==</span><span class="pln"> label_value</span><span class="pun">:</span></code></li><li class="L3"><code class="language-python"><span class="pln">                label_vec</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="lit">0.9</span><span class="pun">)</span></code></li><li class="L4"><code class="language-python"><span class="pln">            </span><span class="kwd">else</span><span class="pun">:</span></code></li><li class="L5"><code class="language-python"><span class="pln">                label_vec</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="lit">0.1</span><span class="pun">)</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> label_vec</span></code></li><li class="L7"><code class="language-python"></code></li><li class="L8"><code class="language-python"></code></li><li class="L9"><code class="language-python"><span class="kwd">def</span><span class="pln"> get_training_data_set</span><span class="pun">():</span></code></li><li class="L0"><code class="language-python"><span class="pln">    </span><span class="str">'''</span></code></li><li class="L1"><code class="language-python"><span class="str">    获得训练数据集</span></code></li><li class="L2"><code class="language-python"><span class="str">    '''</span></code></li><li class="L3"><code class="language-python"><span class="pln">    image_loader </span><span class="pun">=</span><span class="pln"> </span><span class="typ">ImageLoader</span><span class="pun">(</span><span class="str">'train-images-idx3-ubyte'</span><span class="pun">,</span><span class="pln"> </span><span class="lit">60000</span><span class="pun">)</span></code></li><li class="L4"><code class="language-python"><span class="pln">    label_loader </span><span class="pun">=</span><span class="pln"> </span><span class="typ">LabelLoader</span><span class="pun">(</span><span class="str">'train-labels-idx1-ubyte'</span><span class="pun">,</span><span class="pln"> </span><span class="lit">60000</span><span class="pun">)</span></code></li><li class="L5"><code class="language-python"><span class="pln">    </span><span class="kwd">return</span><span class="pln"> image_loader</span><span class="pun">.</span><span class="pln">load</span><span class="pun">(),</span><span class="pln"> label_loader</span><span class="pun">.</span><span class="pln">load</span><span class="pun">()</span></code></li><li class="L6"><code class="language-python"></code></li><li class="L7"><code class="language-python"></code></li><li class="L8"><code class="language-python"><span class="kwd">def</span><span class="pln"> get_test_data_set</span><span class="pun">():</span></code></li><li class="L9"><code class="language-python"><span class="pln">    </span><span class="str">'''</span></code></li><li class="L0"><code class="language-python"><span class="str">    获得测试数据集</span></code></li><li class="L1"><code class="language-python"><span class="str">    '''</span></code></li><li class="L2"><code class="language-python"><span class="pln">    image_loader </span><span class="pun">=</span><span class="pln"> </span><span class="typ">ImageLoader</span><span class="pun">(</span><span class="str">'t10k-images-idx3-ubyte'</span><span class="pun">,</span><span class="pln"> </span><span class="lit">10000</span><span class="pun">)</span></code></li><li class="L3"><code class="language-python"><span class="pln">    label_loader </span><span class="pun">=</span><span class="pln"> </span><span class="typ">LabelLoader</span><span class="pun">(</span><span class="str">'t10k-labels-idx1-ubyte'</span><span class="pun">,</span><span class="pln"> </span><span class="lit">10000</span><span class="pun">)</span></code></li><li class="L4"><code class="language-python"><span class="pln">    </span><span class="kwd">return</span><span class="pln"> image_loader</span><span class="pun">.</span><span class="pln">load</span><span class="pun">(),</span><span class="pln"> label_loader</span><span class="pun">.</span><span class="pln">load</span><span class="pun">()</span></code></li></ol></pre><p data-anchor-id="q8bx">网络的输出是一个10维向量，这个向量第<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-186-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 600.5 494.40013873598514" style="width: 1.452ex; height: 1.089ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use></g></svg></span><script type="math/tex" id="MathJax-Element-186">n</script>个(从0开始编号)元素的值最大，那么<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-187-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -463.20006936799257 600.5 494.40013873598514" style="width: 1.452ex; height: 1.089ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-6E"></use></g></svg></span><script type="math/tex" id="MathJax-Element-187">n</script>就是网络的识别结果。下面是代码实现：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="b2tx" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">def</span><span class="pln"> get_result</span><span class="pun">(</span><span class="pln">vec</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">    max_value_index </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span></code></li><li class="L2"><code class="language-python"><span class="pln">    max_value </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span></code></li><li class="L3"><code class="language-python"><span class="pln">    </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">len</span><span class="pun">(</span><span class="pln">vec</span><span class="pun">)):</span></code></li><li class="L4"><code class="language-python"><span class="pln">        </span><span class="kwd">if</span><span class="pln"> vec</span><span class="pun">[</span><span class="pln">i</span><span class="pun">]</span><span class="pln"> </span><span class="pun">&gt;</span><span class="pln"> max_value</span><span class="pun">:</span></code></li><li class="L5"><code class="language-python"><span class="pln">            max_value </span><span class="pun">=</span><span class="pln"> vec</span><span class="pun">[</span><span class="pln">i</span><span class="pun">]</span></code></li><li class="L6"><code class="language-python"><span class="pln">            max_value_index </span><span class="pun">=</span><span class="pln"> i</span></code></li><li class="L7"><code class="language-python"><span class="pln">    </span><span class="kwd">return</span><span class="pln"> max_value_index</span></code></li></ol></pre><p data-anchor-id="r8nf">我们使用错误率来对网络进行评估，下面是代码实现：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="g9kc" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">def</span><span class="pln"> evaluate</span><span class="pun">(</span><span class="pln">network</span><span class="pun">,</span><span class="pln"> test_data_set</span><span class="pun">,</span><span class="pln"> test_labels</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">    error </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span></code></li><li class="L2"><code class="language-python"><span class="pln">    total </span><span class="pun">=</span><span class="pln"> len</span><span class="pun">(</span><span class="pln">test_data_set</span><span class="pun">)</span></code></li><li class="L3"><code class="language-python"></code></li><li class="L4"><code class="language-python"><span class="pln">    </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">total</span><span class="pun">):</span></code></li><li class="L5"><code class="language-python"><span class="pln">        label </span><span class="pun">=</span><span class="pln"> get_result</span><span class="pun">(</span><span class="pln">test_labels</span><span class="pun">[</span><span class="pln">i</span><span class="pun">])</span></code></li><li class="L6"><code class="language-python"><span class="pln">        predict </span><span class="pun">=</span><span class="pln"> get_result</span><span class="pun">(</span><span class="pln">network</span><span class="pun">.</span><span class="pln">predict</span><span class="pun">(</span><span class="pln">test_data_set</span><span class="pun">[</span><span class="pln">i</span><span class="pun">]))</span></code></li><li class="L7"><code class="language-python"><span class="pln">        </span><span class="kwd">if</span><span class="pln"> label </span><span class="pun">!=</span><span class="pln"> predict</span><span class="pun">:</span></code></li><li class="L8"><code class="language-python"><span class="pln">            error </span><span class="pun">+=</span><span class="pln"> </span><span class="lit">1</span></code></li><li class="L9"><code class="language-python"><span class="pln">    </span><span class="kwd">return</span><span class="pln"> float</span><span class="pun">(</span><span class="pln">error</span><span class="pun">)</span><span class="pln"> </span><span class="pun">/</span><span class="pln"> float</span><span class="pun">(</span><span class="pln">total</span><span class="pun">)</span></code></li></ol></pre><p data-anchor-id="gtos">最后实现我们的训练策略：每训练10轮，评估一次准确率，当准确率开始下降时终止训练。下面是代码实现：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="tq5h" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="kwd">def</span><span class="pln"> train_and_evaluate</span><span class="pun">():</span></code></li><li class="L1"><code class="language-python"><span class="pln">    last_error_ratio </span><span class="pun">=</span><span class="pln"> </span><span class="lit">1.0</span></code></li><li class="L2"><code class="language-python"><span class="pln">    epoch </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span></code></li><li class="L3"><code class="language-python"><span class="pln">    train_data_set</span><span class="pun">,</span><span class="pln"> train_labels </span><span class="pun">=</span><span class="pln"> get_training_data_set</span><span class="pun">()</span></code></li><li class="L4"><code class="language-python"><span class="pln">    test_data_set</span><span class="pun">,</span><span class="pln"> test_labels </span><span class="pun">=</span><span class="pln"> get_test_data_set</span><span class="pun">()</span></code></li><li class="L5"><code class="language-python"><span class="pln">    network </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Network</span><span class="pun">([</span><span class="lit">784</span><span class="pun">,</span><span class="pln"> </span><span class="lit">300</span><span class="pun">,</span><span class="pln"> </span><span class="lit">10</span><span class="pun">])</span></code></li><li class="L6"><code class="language-python"><span class="pln">    </span><span class="kwd">while</span><span class="pln"> </span><span class="kwd">True</span><span class="pun">:</span></code></li><li class="L7"><code class="language-python"><span class="pln">        epoch </span><span class="pun">+=</span><span class="pln"> </span><span class="lit">1</span></code></li><li class="L8"><code class="language-python"><span class="pln">        network</span><span class="pun">.</span><span class="pln">train</span><span class="pun">(</span><span class="pln">train_labels</span><span class="pun">,</span><span class="pln"> train_data_set</span><span class="pun">,</span><span class="pln"> </span><span class="lit">0.3</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">)</span></code></li><li class="L9"><code class="language-python"><span class="pln">        </span><span class="kwd">print</span><span class="pln"> </span><span class="str">'%s epoch %d finished'</span><span class="pln"> </span><span class="pun">%</span><span class="pln"> </span><span class="pun">(</span><span class="pln">now</span><span class="pun">(),</span><span class="pln"> epoch</span><span class="pun">)</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="kwd">if</span><span class="pln"> epoch </span><span class="pun">%</span><span class="pln"> </span><span class="lit">10</span><span class="pln"> </span><span class="pun">==</span><span class="pln"> </span><span class="lit">0</span><span class="pun">:</span></code></li><li class="L1"><code class="language-python"><span class="pln">            error_ratio </span><span class="pun">=</span><span class="pln"> evaluate</span><span class="pun">(</span><span class="pln">network</span><span class="pun">,</span><span class="pln"> test_data_set</span><span class="pun">,</span><span class="pln"> test_labels</span><span class="pun">)</span></code></li><li class="L2"><code class="language-python"><span class="pln">            </span><span class="kwd">print</span><span class="pln"> </span><span class="str">'%s after epoch %d, error ratio is %f'</span><span class="pln"> </span><span class="pun">%</span><span class="pln"> </span><span class="pun">(</span><span class="pln">now</span><span class="pun">(),</span><span class="pln"> epoch</span><span class="pun">,</span><span class="pln"> error_ratio</span><span class="pun">)</span></code></li><li class="L3"><code class="language-python"><span class="pln">            </span><span class="kwd">if</span><span class="pln"> error_ratio </span><span class="pun">&gt;</span><span class="pln"> last_error_ratio</span><span class="pun">:</span></code></li><li class="L4"><code class="language-python"><span class="pln">                </span><span class="kwd">break</span></code></li><li class="L5"><code class="language-python"><span class="pln">            </span><span class="kwd">else</span><span class="pun">:</span></code></li><li class="L6"><code class="language-python"><span class="pln">                last_error_ratio </span><span class="pun">=</span><span class="pln"> error_ratio</span></code></li><li class="L7"><code class="language-python"></code></li><li class="L8"><code class="language-python"></code></li><li class="L9"><code class="language-python"><span class="kwd">if</span><span class="pln"> __name__ </span><span class="pun">==</span><span class="pln"> </span><span class="str">'__main__'</span><span class="pun">:</span></code></li><li class="L0"><code class="language-python"><span class="pln">    train_and_evaluate</span><span class="pun">()</span></code></li></ol></pre><p data-anchor-id="fv58">在我的机器上测试了一下，1个epoch大约需要9000多秒，所以要对代码做很多的性能优化工作（比如用向量化编程）。训练要很久很久，可以把它上传到服务器上，在tmux的session里面去运行。为了防止异常终止导致前功尽弃，我们每训练10轮，就把获得参数值保存在磁盘上，以便后续可以恢复。(代码略)</p><div class="md-section-divider"></div><h2 data-anchor-id="awab" id="向量化编程">向量化编程</h2><blockquote data-anchor-id="ipqg" class="white-blockquote">
  <p>完整代码请参考GitHub: <a href="https://github.com/hanbt/learn_dl/blob/master/fc.py" target="_blank">https://github.com/hanbt/learn_dl/blob/master/fc.py</a> (python2.7)</p>
</blockquote><p data-anchor-id="np7o">在经历了漫长的训练之后，我们可能会想到，肯定有更好的办法！是的，程序员们，现在我们需要告别面向对象编程了，转而去使用另外一种更适合深度学习算法的编程方式：向量化编程。主要有两个原因：一个是我们事实上并不需要真的去定义Node、Connection这样的对象，直接把数学计算实现了就可以了；另一个原因，是底层算法库会针对向量运算做优化（甚至有专用的硬件，比如GPU），程序效率会提升很多。所以，在深度学习的世界里，我们总会想法设法的把计算表达为向量的形式。我相信优秀的程序员不会把自己拘泥于某种（自己熟悉的）编程范式上，而会去学习并使用最为合适的范式。</p><p data-anchor-id="5mkf">下面，我们用向量化编程的方法，重新实现前面的<strong>全连接神经网络</strong>。</p><p data-anchor-id="pvjn">首先，我们需要把所有的计算都表达为向量的形式。对于全连接神经网络来说，主要有三个计算公式。</p><p data-anchor-id="hnkx">前向计算，我们发现<strong>式2</strong>已经是向量化的表达了：</p><div class="md-section-divider"></div><p data-anchor-id="e6eu"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-188-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -904.613537109928 9671.830322580647 1288.3956226069529" style="width: 22.5ex; height: 3.024ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><use href="#MJMAIN-3D" x="807" y="0"></use><use href="#MJMATHI-3C3" x="1863" y="0"></use><use href="#MJMAIN-28" x="2436" y="0"></use><use href="#MJMATHI-57" x="2825" y="0"></use><use href="#MJMAIN-22C5" x="4096" y="0"></use><g transform="translate(4597,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use></g><use href="#MJMAIN-29" x="5169" y="0"></use><use href="#MJMAIN-28" x="7559" y="0"></use><g transform="translate(7948,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-32" x="8781" y="0"></use><use href="#MJMAIN-29" x="9282" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-188">
\vec{a}=\sigma(W\centerdot\vec{x})\qquad (式2)
</script><p></p><p data-anchor-id="uyoj">上式中的<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-189-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -451.20006936799257 572.5 482.40013873598514" style="width: 1.331ex; height: 1.089ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3C3"></use></g></svg></span><script type="math/tex" id="MathJax-Element-189">\sigma</script>表示sigmoid函数。</p><p data-anchor-id="bf7m">反向计算，我们需要把<strong>式3</strong>和<strong>式4</strong>使用向量来表示：</p><div class="md-section-divider"></div><p data-anchor-id="sd08"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true"><span class="MathJax_SVG" id="MathJax-Element-190-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1015.324787109928 44270.673387096766 3740.2967991490177" style="width: 102.823ex; height: 8.71ex; vertical-align: -6.411ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><g transform="translate(15800,0)"><use href="#MJMATHI-3B4" x="9" y="0"></use><use href="#MJMAIN-20D7" x="530" y="281"></use><use href="#MJMAIN-3D" x="808" y="0"></use><g transform="translate(1865,0)"><use href="#MJMATHI-79"></use><use href="#MJMAIN-20D7" x="543" y="7"></use></g><use href="#MJMAIN-28" x="2409" y="0"></use><use href="#MJMAIN-31" x="2799" y="0"></use><use href="#MJMAIN-2212" x="3521" y="0"></use><g transform="translate(4522,0)"><use href="#MJMATHI-79"></use><use href="#MJMAIN-20D7" x="543" y="7"></use></g><use href="#MJMAIN-29" x="5067" y="0"></use><use href="#MJMAIN-28" x="5456" y="0"></use><g transform="translate(5846,0)"><use href="#MJMATHI-74" x="54" y="0"></use><use href="#MJMAIN-20D7" x="553" y="190"></use></g><use href="#MJMAIN-2212" x="6622" y="0"></use><g transform="translate(7623,0)"><use href="#MJMATHI-79"></use><use href="#MJMAIN-20D7" x="543" y="7"></use></g><use href="#MJMAIN-29" x="8167" y="0"></use><use href="#MJMAIN-28" x="10557" y="0"></use><g transform="translate(10946,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-37" x="11780" y="0"></use><use href="#MJMAIN-29" x="12280" y="0"></use></g><g transform="translate(14162,-2342)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g><use href="#MJMAIN-2192" x="157" y="1034"></use><use href="#MJMAIN-3D" x="1593" y="0"></use><g transform="translate(2649,0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><g transform="translate(529,447)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g><use href="#MJMAIN-28" x="4041" y="0"></use><use href="#MJMAIN-31" x="4430" y="0"></use><use href="#MJMAIN-2212" x="5153" y="0"></use><g transform="translate(6153,0)"><use href="#MJMATHI-61"></use><use href="#MJMAIN-20D7" x="499" y="5"></use><g transform="translate(529,447)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g><use href="#MJMAIN-29" x="7545" y="0"></use><g transform="translate(7934,0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="1526" y="583"></use></g><g transform="translate(9612,0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,412)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-2B" x="688" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-31" x="1466" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="1966" y="0"></use></g></g><use href="#MJMAIN-28" x="13832" y="0"></use><g transform="translate(14222,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-38" x="15055" y="0"></use><use href="#MJMAIN-29" x="15556" y="0"></use></g></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-190">
\vec{\delta}=\vec{y}(1-\vec{y})(\vec{t}-\vec{y})\qquad(式7)\\
\vec{\delta^{(l)}}=\vec{a}^{(l)}(1-\vec{a}^{(l)})W^T\delta^{(l+1)}\qquad(式8)
</script><p></p><p data-anchor-id="bk60">在<strong>式8</strong>中，<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-191-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -913.1292620390899 1315.557556728505 943.3293314070825" style="width: 3.024ex; height: 2.177ex; vertical-align: -0.121ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-3B4"></use><g transform="translate(453,362)"><use transform="scale(0.7071067811865476)" href="#MJMAIN-28"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6C" x="389" y="0"></use><use transform="scale(0.7071067811865476)" href="#MJMAIN-29" x="688" y="0"></use></g></g></svg></span><script type="math/tex" id="MathJax-Element-191">\delta^{(l)}</script>表示第l层的误差项；<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-192-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -861.5104670124718 1677.9067273459227 903.7105363804644" style="width: 3.871ex; height: 2.056ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="1526" y="513"></use></g></svg></span><script type="math/tex" id="MathJax-Element-192">W^T</script>表示矩阵<span class="MathJax_Preview"></span><span class="MathJax_SVG" id="MathJax-Element-193-Frame" role="textbox" aria-readonly="true" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -703.2000693679926 1048.5 745.4001387359851" style="width: 2.419ex; height: 1.694ex; vertical-align: -0.242ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use></g></svg></span><script type="math/tex" id="MathJax-Element-193">W</script>的转置。</p><p data-anchor-id="bfcx">我们还需要权重数组W和偏置项b的梯度计算的向量化表示。也就是需要把<strong>式5</strong>使用向量化表示：</p><div class="md-section-divider"></div><p data-anchor-id="7dj0"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-194-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -904.613537109928 12144.972690238306 1288.3956226069529" style="width: 28.185ex; height: 3.024ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g><use href="#MJMAIN-2190" x="1630" y="0"></use><g transform="translate(2908,0)"><use href="#MJMATHI-77"></use><g transform="translate(716,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-2B" x="4483" y="0"></use><use href="#MJMATHI-3B7" x="5483" y="0"></use><g transform="translate(5987,0)"><use href="#MJMATHI-3B4"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A" x="628" y="-213"></use></g><g transform="translate(6823,0)"><use href="#MJMATHI-78"></use><g transform="translate(572,-150)"><use transform="scale(0.7071067811865476)" href="#MJMATHI-6A"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-69" x="412" y="0"></use></g></g><use href="#MJMAIN-28" x="10032" y="0"></use><g transform="translate(10421,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-35" x="11254" y="0"></use><use href="#MJMAIN-29" x="11755" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-194">
w_{ji}\gets w_{ji}+\eta\delta_jx_{ji}\qquad(式5)
</script><p></p><p data-anchor-id="b79n">其对应的向量化表示为：</p><div class="md-section-divider"></div><p data-anchor-id="2o5w"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-195-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1015.324787109928 11194.162049926568 1399.1068726069527" style="width: 26.008ex; height: 3.266ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-57"></use><use href="#MJMAIN-2190" x="1326" y="0"></use><use href="#MJMATHI-57" x="2604" y="0"></use><use href="#MJMAIN-2B" x="3875" y="0"></use><use href="#MJMATHI-3B7" x="4876" y="0"></use><g transform="translate(5379,0)"><use href="#MJMATHI-3B4" x="9" y="0"></use><use href="#MJMAIN-20D7" x="530" y="281"></use></g><g transform="translate(5910,0)"><use href="#MJMATHI-78"></use><use href="#MJMAIN-20D7" x="549" y="6"></use><use transform="scale(0.7071067811865476)" href="#MJMATHI-54" x="809" y="633"></use></g><use href="#MJMAIN-28" x="9081" y="0"></use><g transform="translate(9470,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><use href="#MJMAIN-39" x="10304" y="0"></use><use href="#MJMAIN-29" x="10804" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-195">
W \gets W + \eta\vec{\delta}\vec{x}^T\qquad(式9)
</script><p></p><p data-anchor-id="ho9m">更新偏置项的向量化表示为：</p><div class="md-section-divider"></div><p data-anchor-id="t8dw"><span class="MathJax_Preview"></span></p><div class="MathJax_SVG_Display" role="textbox" aria-readonly="true" style="text-align: center;"><span class="MathJax_SVG" id="MathJax-Element-196-Frame" style="font-size: 100%; display: inline-block;"><svg xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 -1015.324787109928 9369.005322580644 1399.1068726069527" style="width: 21.774ex; height: 3.266ex; vertical-align: -0.968ex; margin: 1px 0px;"><g stroke="black" fill="black" stroke-width="0" transform="matrix(1 0 0 -1 0 0)"><use href="#MJMATHI-62" x="20" y="0"></use><use href="#MJMAIN-20D7" x="470" y="258"></use><use href="#MJMAIN-2190" x="748" y="0"></use><g transform="translate(2027,0)"><use href="#MJMATHI-62" x="20" y="0"></use><use href="#MJMAIN-20D7" x="470" y="258"></use></g><use href="#MJMAIN-2B" x="2720" y="0"></use><use href="#MJMATHI-3B7" x="3721" y="0"></use><g transform="translate(4224,0)"><use href="#MJMATHI-3B4" x="9" y="0"></use><use href="#MJMAIN-20D7" x="530" y="281"></use></g><use href="#MJMAIN-28" x="6755" y="0"></use><g transform="translate(7145,0)"><text font-family="STIXGeneral,&#39;Arial Unicode MS&#39;,serif" font-style="" font-weight="" stroke="none" transform="scale(52.08314516129032) matrix(1 0 0 -1 0 0)">式</text></g><g transform="translate(7978,0)"><use href="#MJMAIN-31"></use><use href="#MJMAIN-30" x="500" y="0"></use></g><use href="#MJMAIN-29" x="8979" y="0"></use></g></svg></span></div><script type="math/tex; mode=display" id="MathJax-Element-196">
\vec{b} \gets \vec{b} + \eta\vec{\delta}\qquad(式10)
</script><p></p><p data-anchor-id="sjul">现在，我们根据上面几个公式，重新实现一个类：FullConnectedLayer。它实现了全连接层的前向和后向计算：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="u3f0" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="com"># 全连接层实现类</span></code></li><li class="L1"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">FullConnectedLayer</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> input_size</span><span class="pun">,</span><span class="pln"> output_size</span><span class="pun">,</span><span class="pln"> </span></code></li><li class="L3"><code class="language-python"><span class="pln">                 activator</span><span class="pun">):</span></code></li><li class="L4"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L5"><code class="language-python"><span class="str">        构造函数</span></code></li><li class="L6"><code class="language-python"><span class="str">        input_size: 本层输入向量的维度</span></code></li><li class="L7"><code class="language-python"><span class="str">        output_size: 本层输出向量的维度</span></code></li><li class="L8"><code class="language-python"><span class="str">        activator: 激活函数</span></code></li><li class="L9"><code class="language-python"><span class="str">        '''</span></code></li><li class="L0"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">input_size </span><span class="pun">=</span><span class="pln"> input_size</span></code></li><li class="L1"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">output_size </span><span class="pun">=</span><span class="pln"> output_size</span></code></li><li class="L2"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">activator </span><span class="pun">=</span><span class="pln"> activator</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="com"># 权重数组W</span></code></li><li class="L4"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">W </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">random</span><span class="pun">.</span><span class="pln">uniform</span><span class="pun">(-</span><span class="lit">0.1</span><span class="pun">,</span><span class="pln"> </span><span class="lit">0.1</span><span class="pun">,</span></code></li><li class="L5"><code class="language-python"><span class="pln">            </span><span class="pun">(</span><span class="pln">output_size</span><span class="pun">,</span><span class="pln"> input_size</span><span class="pun">))</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="com"># 偏置项b</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">b </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">zeros</span><span class="pun">((</span><span class="pln">output_size</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">))</span></code></li><li class="L8"><code class="language-python"><span class="pln">        </span><span class="com"># 输出向量</span></code></li><li class="L9"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">output </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">zeros</span><span class="pun">((</span><span class="pln">output_size</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">))</span></code></li><li class="L0"><code class="language-python"></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> forward</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> input_array</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        前向计算</span></code></li><li class="L4"><code class="language-python"><span class="str">        input_array: 输入向量，维度必须等于input_size</span></code></li><li class="L5"><code class="language-python"><span class="str">        '''</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="com"># 式2</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">input </span><span class="pun">=</span><span class="pln"> input_array</span></code></li><li class="L8"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">output </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">activator</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span></code></li><li class="L9"><code class="language-python"><span class="pln">            np</span><span class="pun">.</span><span class="pln">dot</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">W</span><span class="pun">,</span><span class="pln"> input_array</span><span class="pun">)</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">b</span><span class="pun">)</span></code></li><li class="L0"><code class="language-python"></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> backward</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> delta_array</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        反向计算W和b的梯度</span></code></li><li class="L4"><code class="language-python"><span class="str">        delta_array: 从上一层传递过来的误差项</span></code></li><li class="L5"><code class="language-python"><span class="str">        '''</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="com"># 式8</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">delta </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">activator</span><span class="pun">.</span><span class="pln">backward</span><span class="pun">(</span><span class="pln">self</span><span class="pun">.</span><span class="pln">input</span><span class="pun">)</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">dot</span><span class="pun">(</span></code></li><li class="L8"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">W</span><span class="pun">.</span><span class="pln">T</span><span class="pun">,</span><span class="pln"> delta_array</span><span class="pun">)</span></code></li><li class="L9"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">W_grad </span><span class="pun">=</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">dot</span><span class="pun">(</span><span class="pln">delta_array</span><span class="pun">,</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">input</span><span class="pun">.</span><span class="pln">T</span><span class="pun">)</span></code></li><li class="L0"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">b_grad </span><span class="pun">=</span><span class="pln"> delta_array</span></code></li><li class="L1"><code class="language-python"></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> update</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> learning_rate</span><span class="pun">):</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L4"><code class="language-python"><span class="str">        使用梯度下降算法更新权重</span></code></li><li class="L5"><code class="language-python"><span class="str">        '''</span></code></li><li class="L6"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">W </span><span class="pun">+=</span><span class="pln"> learning_rate </span><span class="pun">*</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">W_grad</span></code></li><li class="L7"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">b </span><span class="pun">+=</span><span class="pln"> learning_rate </span><span class="pun">*</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">b_grad</span></code></li></ol></pre><p data-anchor-id="36rt">上面这个类一举取代了原先的Layer、Node、Connection等类，不但代码更加容易理解，而且运行速度也快了几百倍。</p><p data-anchor-id="oof6">现在，我们对Network类稍作修改，使之用到FullConnectedLayer：</p><div class="md-section-divider"></div><pre class="prettyprint linenums prettyprinted" data-anchor-id="vdt5" style=""><ol class="linenums"><li class="L0"><code class="language-python"><span class="com"># Sigmoid激活函数类</span></code></li><li class="L1"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">SigmoidActivator</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> forward</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> weighted_input</span><span class="pun">):</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> </span><span class="lit">1.0</span><span class="pln"> </span><span class="pun">/</span><span class="pln"> </span><span class="pun">(</span><span class="lit">1.0</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> np</span><span class="pun">.</span><span class="pln">exp</span><span class="pun">(-</span><span class="pln">weighted_input</span><span class="pun">))</span></code></li><li class="L4"><code class="language-python"></code></li><li class="L5"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> backward</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> output</span><span class="pun">):</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> output </span><span class="pun">*</span><span class="pln"> </span><span class="pun">(</span><span class="lit">1</span><span class="pln"> </span><span class="pun">-</span><span class="pln"> output</span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"></code></li><li class="L8"><code class="language-python"></code></li><li class="L9"><code class="language-python"><span class="com"># 神经网络类</span></code></li><li class="L0"><code class="language-python"><span class="kwd">class</span><span class="pln"> </span><span class="typ">Network</span><span class="pun">(</span><span class="pln">object</span><span class="pun">):</span></code></li><li class="L1"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> layers</span><span class="pun">):</span></code></li><li class="L2"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L3"><code class="language-python"><span class="str">        构造函数</span></code></li><li class="L4"><code class="language-python"><span class="str">        '''</span></code></li><li class="L5"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">layers </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">len</span><span class="pun">(</span><span class="pln">layers</span><span class="pun">)</span><span class="pln"> </span><span class="pun">-</span><span class="pln"> </span><span class="lit">1</span><span class="pun">):</span></code></li><li class="L7"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span></code></li><li class="L8"><code class="language-python"><span class="pln">                </span><span class="typ">FullConnectedLayer</span><span class="pun">(</span></code></li><li class="L9"><code class="language-python"><span class="pln">                    layers</span><span class="pun">[</span><span class="pln">i</span><span class="pun">],</span><span class="pln"> layers</span><span class="pun">[</span><span class="pln">i</span><span class="pun">+</span><span class="lit">1</span><span class="pun">],</span></code></li><li class="L0"><code class="language-python"><span class="pln">                    </span><span class="typ">SigmoidActivator</span><span class="pun">()</span></code></li><li class="L1"><code class="language-python"><span class="pln">                </span><span class="pun">)</span></code></li><li class="L2"><code class="language-python"><span class="pln">            </span><span class="pun">)</span></code></li><li class="L3"><code class="language-python"></code></li><li class="L4"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> predict</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> sample</span><span class="pun">):</span></code></li><li class="L5"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L6"><code class="language-python"><span class="str">        使用神经网络实现预测</span></code></li><li class="L7"><code class="language-python"><span class="str">        sample: 输入样本</span></code></li><li class="L8"><code class="language-python"><span class="str">        '''</span></code></li><li class="L9"><code class="language-python"><span class="pln">        output </span><span class="pun">=</span><span class="pln"> sample</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> layer </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">:</span></code></li><li class="L1"><code class="language-python"><span class="pln">            layer</span><span class="pun">.</span><span class="pln">forward</span><span class="pun">(</span><span class="pln">output</span><span class="pun">)</span></code></li><li class="L2"><code class="language-python"><span class="pln">            output </span><span class="pun">=</span><span class="pln"> layer</span><span class="pun">.</span><span class="pln">output</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> output</span></code></li><li class="L4"><code class="language-python"></code></li><li class="L5"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> train</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> labels</span><span class="pun">,</span><span class="pln"> data_set</span><span class="pun">,</span><span class="pln"> rate</span><span class="pun">,</span><span class="pln"> epoch</span><span class="pun">):</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="str">'''</span></code></li><li class="L7"><code class="language-python"><span class="str">        训练函数</span></code></li><li class="L8"><code class="language-python"><span class="str">        labels: 样本标签</span></code></li><li class="L9"><code class="language-python"><span class="str">        data_set: 输入样本</span></code></li><li class="L0"><code class="language-python"><span class="str">        rate: 学习速率</span></code></li><li class="L1"><code class="language-python"><span class="str">        epoch: 训练轮数</span></code></li><li class="L2"><code class="language-python"><span class="str">        '''</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">epoch</span><span class="pun">):</span></code></li><li class="L4"><code class="language-python"><span class="pln">            </span><span class="kwd">for</span><span class="pln"> d </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="pln">len</span><span class="pun">(</span><span class="pln">data_set</span><span class="pun">)):</span></code></li><li class="L5"><code class="language-python"><span class="pln">                self</span><span class="pun">.</span><span class="pln">train_one_sample</span><span class="pun">(</span><span class="pln">labels</span><span class="pun">[</span><span class="pln">d</span><span class="pun">],</span><span class="pln"> </span></code></li><li class="L6"><code class="language-python"><span class="pln">                    data_set</span><span class="pun">[</span><span class="pln">d</span><span class="pun">],</span><span class="pln"> rate</span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"></code></li><li class="L8"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> train_one_sample</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> label</span><span class="pun">,</span><span class="pln"> sample</span><span class="pun">,</span><span class="pln"> rate</span><span class="pun">):</span></code></li><li class="L9"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">predict</span><span class="pun">(</span><span class="pln">sample</span><span class="pun">)</span></code></li><li class="L0"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">calc_gradient</span><span class="pun">(</span><span class="pln">label</span><span class="pun">)</span></code></li><li class="L1"><code class="language-python"><span class="pln">        self</span><span class="pun">.</span><span class="pln">update_weight</span><span class="pun">(</span><span class="pln">rate</span><span class="pun">)</span></code></li><li class="L2"><code class="language-python"></code></li><li class="L3"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> calc_gradient</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> label</span><span class="pun">):</span></code></li><li class="L4"><code class="language-python"><span class="pln">        delta </span><span class="pun">=</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[-</span><span class="lit">1</span><span class="pun">].</span><span class="pln">activator</span><span class="pun">.</span><span class="pln">backward</span><span class="pun">(</span></code></li><li class="L5"><code class="language-python"><span class="pln">            self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[-</span><span class="lit">1</span><span class="pun">].</span><span class="pln">output</span></code></li><li class="L6"><code class="language-python"><span class="pln">        </span><span class="pun">)</span><span class="pln"> </span><span class="pun">*</span><span class="pln"> </span><span class="pun">(</span><span class="pln">label </span><span class="pun">-</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[-</span><span class="lit">1</span><span class="pun">].</span><span class="pln">output</span><span class="pun">)</span></code></li><li class="L7"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> layer </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">[::-</span><span class="lit">1</span><span class="pun">]:</span></code></li><li class="L8"><code class="language-python"><span class="pln">            layer</span><span class="pun">.</span><span class="pln">backward</span><span class="pun">(</span><span class="pln">delta</span><span class="pun">)</span></code></li><li class="L9"><code class="language-python"><span class="pln">            delta </span><span class="pun">=</span><span class="pln"> layer</span><span class="pun">.</span><span class="pln">delta</span></code></li><li class="L0"><code class="language-python"><span class="pln">        </span><span class="kwd">return</span><span class="pln"> delta</span></code></li><li class="L1"><code class="language-python"></code></li><li class="L2"><code class="language-python"><span class="pln">    </span><span class="kwd">def</span><span class="pln"> update_weight</span><span class="pun">(</span><span class="pln">self</span><span class="pun">,</span><span class="pln"> rate</span><span class="pun">):</span></code></li><li class="L3"><code class="language-python"><span class="pln">        </span><span class="kwd">for</span><span class="pln"> layer </span><span class="kwd">in</span><span class="pln"> self</span><span class="pun">.</span><span class="pln">layers</span><span class="pun">:</span></code></li><li class="L4"><code class="language-python"><span class="pln">            layer</span><span class="pun">.</span><span class="pln">update</span><span class="pun">(</span><span class="pln">rate</span><span class="pun">)</span></code></li></ol></pre><p data-anchor-id="u1z2">现在，Network类也清爽多了，用我们的新代码再次训练一下MNIST数据集吧。</p><div class="md-section-divider"></div><h2 data-anchor-id="bosv" id="小结">小结</h2><p data-anchor-id="w5ai">至此，你已经完成了又一次漫长的学习之旅。你现在应该已经明白了神经网络的基本原理，高兴的话，你甚至有能力去动手实现一个，并用它解决一些问题。如果感到困难也不要气馁，这篇文章是一个重要的分水岭，如果你完全弄明白了的话，在真正的『小白』和装腔作势的『大牛』面前吹吹牛是完全没有问题的。</p><p data-anchor-id="xemt">作为深度学习入门的系列文章，本文也是上半场的结束。在这个半场，你掌握了机器学习、神经网络的<strong>基本</strong>概念，并且有能力去动手解决一些简单的问题（例如手写数字识别，如果用传统的观点来看，其实这些问题也不简单）。而且，一旦掌握基本概念，后面的学习就容易多了。</p><p data-anchor-id="hau2">在下半场，我们讲介绍更多『深度』学习的内容，我们已经讲了神经网络(Neutrol Network)，但是并没有讲深度神经网络(Deep Neutrol Network)。Deep会带来更加强大的能力，同时也带来更多的问题。如果不理解这些问题和它们的解决方案，也不能说你入门了『深度』学习。</p><p data-anchor-id="vqg3">目前业界有很多开源的神经网络实现，它们的功能也要强大的多，因此你并不需要事必躬亲的去实现自己的神经网络。我们在上半场不断的从头发明轮子，是为了让你明白神经网络的基本原理，这样你就能非常迅速的掌握这些工具。在下半场的文章中，我们改变了策略：不会再去从头开始去实现，而是尽可能应用现有的工具。</p><p data-anchor-id="v1dw">下一篇文章，我们介绍不同结构的神经网络，比如鼎鼎大名的<strong>卷积神经网络</strong>，它在图像和语音领域已然创造了诸多奇迹，在自然语言处理领域的研究也如火如荼。某种意义上说，它的成功大大提升了人们对于深度学习的信心。</p><p data-anchor-id="65d8">好了，同学们累了吧，奉上美图一张，放松一下心情！</p><p data-anchor-id="sc1q"><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/2256672-71d99cc6e90c9400.jpg" alt="" title=""></p><div class="md-section-divider"></div><h2 data-anchor-id="rcw5" id="参考资料">参考资料</h2><ol data-anchor-id="vit7">
<li>Tom M. Mitchell, "机器学习", 曾华军等译, 机械工业出版社</li>
<li>CS 224N / Ling 284, Neural Networks for Named Entity Recognition</li>
<li>LeCun et al. Gradient-Based Learning Applied to Document Recognition 1998</li>
</ol></div>
    <div class="remark-icons"><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="m46z">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="f0rd">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 255px; left: 1350px; display: none;" data-anchor-id="41in">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="x7r0">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="xuc5">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="uel4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="l2q6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 1142.75px; left: 1350px; display: none;" data-anchor-id="tyid">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 1312.75px; left: 1350px; display: none;" data-anchor-id="lt93">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 1372.75px; left: 1350px; display: none;" data-anchor-id="jhtz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 1446.75px; left: 1350px; display: none;" data-anchor-id="hgru">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 1643.75px; left: 1350px; display: none;" data-anchor-id="l6hf">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="c4p4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 1765.75px; left: 1350px; display: none;" data-anchor-id="v3lw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="di7d">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="afzy">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="1mv4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qt2s">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 2037.8px; left: 1350px; display: none;" data-anchor-id="yvzv">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 2297.8px; left: 1350px; display: none;" data-anchor-id="4tgn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="9wwr">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 2420.83px; left: 1350px; display: none;" data-anchor-id="9p0k">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="jhuu">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 2569.83px; left: 1350px; display: none;" data-anchor-id="7x2j">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 2939.83px; left: 1350px; display: none;" data-anchor-id="ft98">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="7uzv">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="t4h2">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="f526">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="zmso">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="7sfa">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 3428.83px; left: 1350px; display: none;" data-anchor-id="4eh9">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="xqw3">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 3576.83px; left: 1350px; display: none;" data-anchor-id="zp36">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 3862.83px; left: 1350px; display: none;" data-anchor-id="ykzb">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 3990.83px; left: 1350px; display: none;" data-anchor-id="frle">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 4118.83px; left: 1350px; display: none;" data-anchor-id="o1kt">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="cv1g">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 4235.84px; left: 1350px; display: none;" data-anchor-id="11vf">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 4309.84px; left: 1350px; display: none;" data-anchor-id="cq60">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="nvvo">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="isur">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 4590.91px; left: 1350px; display: none;" data-anchor-id="m8jq">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="3sfs">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="kz35">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="74ly">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="h72c">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="vd0n">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="0a6h">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="6a6w">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 5364.16px; left: 1350px; display: none;" data-anchor-id="ai10">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="q4mz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 5603.23px; left: 1350px; display: none;" data-anchor-id="kkyi">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="dpnz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 5697.23px; left: 1350px; display: none;" data-anchor-id="uchd">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="msvf">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="bmpe">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 6168.23px; left: 1350px; display: none;" data-anchor-id="lnrm">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="t61o">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 6333.3px; left: 1350px; display: none;" data-anchor-id="092q">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 6395.3px; left: 1350px; display: none;" data-anchor-id="0454">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 6455.3px; left: 1350px; display: none;" data-anchor-id="lx3f">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 6556.3px; left: 1350px; display: none;" data-anchor-id="crod">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 6603.3px; left: 1350px; display: none;" data-anchor-id="io4w">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 6663.3px; left: 1350px; display: none;" data-anchor-id="1rb4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 6764.3px; left: 1350px; display: none;" data-anchor-id="vbpl">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 6865.3px; left: 1350px; display: none;" data-anchor-id="h9o1">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 6912.3px; left: 1350px; display: none;" data-anchor-id="yv1n">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 7191.3px; left: 1350px; display: none;" data-anchor-id="ygu1">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="1s9t">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="g1kw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ks4m">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="7a0m">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ric6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="4gyz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="6dau">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="amd4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="g81d">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="48s2">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="f819">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="fnuj">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="kmsn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="o3kn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="5tk6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="fym2">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="k0cb">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="hdcf">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="6frf">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="mln1">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="xdxw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2ymm">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="hndg">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="9muo">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ub3y">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qxcc">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="d5sd">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="z1dj">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="wydc">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="1b6r">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="kmvo">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qrep">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="f48g">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="cgqt">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="432c">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qjsy">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="5hl3">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="n0ie">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="zee1">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="dsw7">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="r2nd">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="b2sg">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="rvfo">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="u5gq">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="r0w3">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="xcif">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="l06o">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ta5e">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="js55">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="q9f3">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="0bkw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="c2kt">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="zkzc">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="wbn6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="941o">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="yxfi">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="86e9">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="qd5f">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="714y">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="cmhp">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 11560.1px; left: 1350px; display: none;" data-anchor-id="jqv3">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="f8dg">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 11763.1px; left: 1350px; display: none;" data-anchor-id="4vka">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 11828.1px; left: 1350px; display: none;" data-anchor-id="fpq5">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 11902.1px; left: 1350px; display: none;" data-anchor-id="rduh">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 11949.1px; left: 1350px; display: none;" data-anchor-id="5fb1">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="zxn4">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 12321.1px; left: 1350px; display: none;" data-anchor-id="om70">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="uvcw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 12513.1px; left: 1350px; display: none;" data-anchor-id="27j9">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="4gwx">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 13939.1px; left: 1350px; display: none;" data-anchor-id="y073">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="hue9">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 14725.1px; left: 1350px; display: none;" data-anchor-id="b3tz">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="xwmo">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 15471.1px; left: 1350px; display: none;" data-anchor-id="qxd8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="mdf6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="xha7">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="fbfg">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 16663.1px; left: 1350px; display: none;" data-anchor-id="03nu">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="zp3w">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="kwer">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 18709.1px; left: 1350px; display: none;" data-anchor-id="umv1">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="3tcx">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="thsl">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="4x72">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 18999.1px; left: 1350px; display: none;" data-anchor-id="8rxv">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="004m">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 19119.1px; left: 1350px; display: none;" data-anchor-id="cztu">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="fy0k">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2mo8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="9cc8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="e7zn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 19473.2px; left: 1350px; display: none;" data-anchor-id="9vkf">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="naur">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="wcu1">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 19685.2px; left: 1350px; display: none;" data-anchor-id="3bdm">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="5tf6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="pvb0">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="f94n">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="6acg">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 20714.2px; left: 1350px; display: none;" data-anchor-id="5e7q">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="394n">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 20875.2px; left: 1350px; display: none;" data-anchor-id="vwro">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 20976.2px; left: 1350px; display: none;" data-anchor-id="0pr1">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 21050.2px; left: 1350px; display: none;" data-anchor-id="zx0q">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="nv8r">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="h3b2">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="a5gs">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="levm">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="t39p">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="fist">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="l63w">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="3xym">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="zmnu">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 21883.4px; left: 1350px; display: none;" data-anchor-id="61xm">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="g8cr">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2au6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="q8bx">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="b2tx">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="r8nf">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="g9kc">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="gtos">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="tq5h">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="fv58">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="awab">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ipqg">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="np7o">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="5mkf">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="pvjn">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="hnkx">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="e6eu">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="uyoj">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="bf7m">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="sd08">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="bk60">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="bfcx">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="7dj0">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="b79n">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="2o5w">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="ho9m">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="t8dw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="sjul">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="u3f0">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="36rt">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="oof6">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="vdt5">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="u1z2">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="bosv">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="w5ai">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="xemt">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="hau2">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="vqg3">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="v1dw">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="65d8">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="top: 29720.4px; left: 1350px; display: none;" data-anchor-id="sc1q">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count">+</span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="rcw5">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div><div class="remark-icon unselectable remark-icon-empty" style="display: none;" data-anchor-id="vit7">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div></div>
</div>

<!--in page preview buttons. -->
<div class="in-page-preview-buttons in-page-preview-buttons-full-reader">
    <ul>
        <li class="in-page-button dropdown" id="preview-toc-button" title="内容目录 Ctrl+Alt+O">
            <span class="dropdown-toggle icon-list" data-toggle="dropdown" style="color: rgba(102, 128, 153, 0.45);"></span>
            <div id="toc-list" class="dropdown-menu theme pull-right theme-white"> <!-- Add theme means this element will be changed when apply theme color. -->
                <h3>内容目录</h3>
                <hr>
                <div class="table-of-contents"><div class="toc">
<ul>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E9%9B%B6%E5%9F%BA%E7%A1%80%E5%85%A5%E9%97%A8%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A03-%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E5%92%8C%E5%8F%8D%E5%90%91%E4%BC%A0%E6%92%AD%E7%AE%97%E6%B3%95">零基础入门深度学习(3) - 神经网络和反向传播算法</a><ul>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E6%96%87%E7%AB%A0%E5%88%97%E8%A1%A8">文章列表</a></li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E5%BE%80%E6%9C%9F%E5%9B%9E%E9%A1%BE">往期回顾</a></li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E7%A5%9E%E7%BB%8F%E5%85%83">神经元</a></li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E6%98%AF%E5%95%A5">神经网络是啥</a></li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E8%AE%A1%E7%AE%97%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%9A%84%E8%BE%93%E5%87%BA">计算神经网络的输出</a><ul>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%9A%84%E7%9F%A9%E9%98%B5%E8%A1%A8%E7%A4%BA">神经网络的矩阵表示</a></li>
</ul>
</li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%9A%84%E8%AE%AD%E7%BB%83">神经网络的训练</a><ul>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E5%8F%8D%E5%90%91%E4%BC%A0%E6%92%AD%E7%AE%97%E6%B3%95back-propagation">反向传播算法(Back Propagation)</a></li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E5%8F%8D%E5%90%91%E4%BC%A0%E6%92%AD%E7%AE%97%E6%B3%95%E7%9A%84%E6%8E%A8%E5%AF%BC">反向传播算法的推导</a><ul>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E8%BE%93%E5%87%BA%E5%B1%82%E6%9D%83%E5%80%BC%E8%AE%AD%E7%BB%83">输出层权值训练</a></li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E9%9A%90%E8%97%8F%E5%B1%82%E6%9D%83%E5%80%BC%E8%AE%AD%E7%BB%83">隐藏层权值训练</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%9A%84%E5%AE%9E%E7%8E%B0">神经网络的实现</a><ul>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E6%A2%AF%E5%BA%A6%E6%A3%80%E6%9F%A5">梯度检查</a></li>
</ul>
</li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E5%AE%9E%E6%88%98%E6%89%8B%E5%86%99%E6%95%B0%E5%AD%97%E8%AF%86%E5%88%AB">神经网络实战——手写数字识别</a><ul>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E8%B6%85%E5%8F%82%E6%95%B0%E7%9A%84%E7%A1%AE%E5%AE%9A">超参数的确定</a></li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E6%A8%A1%E5%9E%8B%E7%9A%84%E8%AE%AD%E7%BB%83%E5%92%8C%E8%AF%84%E4%BC%B0">模型的训练和评估</a></li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E4%BB%A3%E7%A0%81%E5%AE%9E%E7%8E%B0">代码实现</a></li>
</ul>
</li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E5%90%91%E9%87%8F%E5%8C%96%E7%BC%96%E7%A8%8B">向量化编程</a></li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E5%B0%8F%E7%BB%93">小结</a></li>
<li><a href="https://www.zybuluo.com/hanbingtao/note/476663#%E5%8F%82%E8%80%83%E8%B5%84%E6%96%99">参考资料</a></li>
</ul>
</li>
</ul>
</div>
</div>
            </div>
        </li>
    </ul>
</div>

<div id="reader-full-toolbar" class="reader-full-toolbar-shown" style="padding-top: 0;">
    <ul id="reader-full-toolbar-home" class="preview-button-row">
        <li class="preview-button-full-reader" id="preview-editor-button" title="撰写文本 Ctrl+Alt+M">
            <span class="icon-pencil" style="color: rgb(187, 187, 187);"></span>
        </li>
    </ul>
    <ul id="preview-button-row" class="preview-button-row">
        <li class="preview-button-full-reader dropdown" id="preview-list-button" title="文本列表 Ctrl+Alt+F">
            <span class="dropdown-toggle icon-reorder" data-toggle="dropdown" style="color: rgb(187, 187, 187);"></span>
            <ul id="file-list" class="dropdown-menu theme-black pull-right" role="menu" style="max-height: 652px;">
                    <li>
                    <ul class="tag-list">
                        <li class="tag-item item" tag-name="机器学习">
                            <span class="pull-left"><i class="icon-tag"></i><span class="tag-name">机器学习</span></span>
                            <span class="tag-count pull-right">7</span>
                            <div class="clearfix"></div>
                        </li>
                            
    <li class="file-item item" file-created-date="2017-02-28 00:59:57" style="display: list-item;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/626300" title="【已发布】 2017-08-29 23:41">
        <i class="icon-share-sign"></i>
        <span id="626300">零基础入门深度学习(7) - 递归神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2017-01-08 23:28:36" style="display: list-item;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/581764" title="【已发布】 2017-08-28 19:55">
        <i class="icon-share-sign"></i>
        <span id="581764">零基础入门深度学习(6) - 长短时记忆网络(LSTM)</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-11-05 19:16:51" style="display: list-item;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/541458" title="【已发布】 2017-08-28 19:54">
        <i class="icon-share-sign"></i>
        <span id="541458">零基础入门深度学习(5) - 循环神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-10-09 20:30:46" style="display: list-item;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/485480" title="【已发布】 2017-08-28 19:53">
        <i class="icon-share-sign"></i>
        <span id="485480">零基础入门深度学习(4) - 卷积神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-08-24 21:39:25" style="display: list-item;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/476663" title="【已发布】 2017-10-17 22:25">
        <i class="icon-share-sign"></i>
        <span id="476663" class="whiter-on-black">零基础入门深度学习(3) - 神经网络和反向传播算法</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-26 01:44:30" style="display: list-item;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/448086" title="【已发布】 2017-08-28 19:40">
        <i class="icon-share-sign"></i>
        <span id="448086">零基础入门深度学习(2) - 线性单元和梯度下降</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-12 11:10:42" style="display: list-item;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/433855" title="【已发布】 2017-08-28 19:35">
        <i class="icon-share-sign"></i>
        <span id="433855">零基础入门深度学习(1) - 感知器</span>
        </a>
    </li>

                    </ul>
                    </li>
                    <li>
                    <ul class="tag-list">
                        <li class="tag-item item" tag-name="深度学习入门">
                            <span class="pull-left"><i class="icon-tag"></i><span class="tag-name">深度学习入门</span></span>
                            <span class="tag-count pull-right">7</span>
                            <div class="clearfix"></div>
                        </li>
                            
    <li class="file-item item" file-created-date="2017-02-28 00:59:57" style="display: none;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/626300" title="【已发布】 2017-08-29 23:41">
        <i class="icon-share-sign"></i>
        <span id="626300">零基础入门深度学习(7) - 递归神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2017-01-08 23:28:36" style="display: none;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/581764" title="【已发布】 2017-08-28 19:55">
        <i class="icon-share-sign"></i>
        <span id="581764">零基础入门深度学习(6) - 长短时记忆网络(LSTM)</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-11-05 19:16:51" style="display: none;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/541458" title="【已发布】 2017-08-28 19:54">
        <i class="icon-share-sign"></i>
        <span id="541458">零基础入门深度学习(5) - 循环神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-10-09 20:30:46" style="display: none;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/485480" title="【已发布】 2017-08-28 19:53">
        <i class="icon-share-sign"></i>
        <span id="485480">零基础入门深度学习(4) - 卷积神经网络</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-08-24 21:39:25" style="display: none;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/476663" title="【已发布】 2017-10-17 22:25">
        <i class="icon-share-sign"></i>
        <span id="476663" class="whiter-on-black">零基础入门深度学习(3) - 神经网络和反向传播算法</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-26 01:44:30" style="display: none;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/448086" title="【已发布】 2017-08-28 19:40">
        <i class="icon-share-sign"></i>
        <span id="448086">零基础入门深度学习(2) - 线性单元和梯度下降</span>
        </a>
    </li>

                            
    <li class="file-item item" file-created-date="2016-07-12 11:10:42" style="display: none;">
        <a tabindex="-1" href="https://www.zybuluo.com/hanbingtao/note/433855" title="【已发布】 2017-08-28 19:35">
        <i class="icon-share-sign"></i>
        <span id="433855">零基础入门深度学习(1) - 感知器</span>
        </a>
    </li>

                    </ul>
                    </li>
            </ul>
            <ul id="file-list-topbar" class="dropdown-menu theme-black pull-right" role="menu">
                <li id="search-file-bar" style="display: block;">
                    <i class="icon-search icon-large"></i>
                    <input type="text" id="search-file-textbox" placeholder="搜索 hanbingtao 的文稿标题， * 显示全部">
                    <i class="icon-level-down icon-rotate-90 icon-large"></i>
                </li>
                <li id="tag-file-bar" style="display: none;">
                    以下【标签】将用于标记这篇文稿：
                </li>
            </ul>
        </li>
        <li class="preview-button-full-reader" id="preview-theme-button" title="主题切换 Ctrl+Alt+Y">
            <span class="icon-adjust" style="color: rgb(187, 187, 187);"></span>
        </li>
        <li class="preview-button-full-reader" id="preview-fullscreen-button" title="全屏模式 F11">
            <span class="icon-fullscreen" style="color: rgb(187, 187, 187);"></span>
        </li>
        <li class="preview-button-full-reader wmd-spacer"></li>
        <li class="preview-button-full-reader dropdown" id="preview-about-button" title="关于本站">
            <span class="dropdown-toggle icon-info-sign" data-toggle="dropdown" data-hover="dropdown" data-delay="100" data-close-others="true" style="color: rgb(187, 187, 187);"></span>
            <ul id="about-menu" class="dropdown-menu theme-black pull-right" role="menu">
                <li title="下载全平台客户端"><a tabindex="-1" href="https://www.zybuluo.com/cmd" target="_blank"><i class="icon-laptop"></i>下载客户端</a></li>
                <li title="@ghosert"><a tabindex="-1" href="http://www.weibo.com/ghosert" target="_blank"><i class="icon-weibo"></i>关注开发者</a></li>
                <li title=""><a tabindex="-1" href="https://github.com/ghosert/cmd-editor/issues" target="_blank"><i class="icon-github-alt"></i>报告问题，建议</a></li>
                <li title="support@zybuluo.com"><a tabindex="-1" href="mailto:support@zybuluo.com" target="_blank"><i class="icon-envelope"></i>联系我们</a></li>
            </ul>
        </li>
    </ul>
</div>
<ul id="reader-full-toolbar-tail" class="reader-full-toolbar-tail-shown">
    <li class="preview-button-full-reader" id="preview-hidden-button" title="隐藏工具栏 Ctrl+Alt+I">
        <span class="icon-chevron-sign-right" style="color: rgb(187, 187, 187);"></span>
    </li>
</ul>






<!-- side remark, hidden when loading. -->
<div class="remark-list side-remark-hidden">
    <div class="remark-items"></div>
    <div class="leave-remark unselectable"><span class="icon-plus-sign-alt"></span><span>添加新批注</span></div>
    <div class="new-remark">
    <div class="remark-head"><a><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/default-head.jpg" style="width: 32px; height: 32px;"></a></div>
    <div class="remark-author unselectable"><strong>markliu2019</strong></div>
    <div class="remark-editor" contenteditable="true" spellcheck="false"></div>
    <!-- this will be filled up by js.
    <div class="inline-error">402/400</div> for new remark
    <div class="inline-error">202/200</div> for new reply
    -->
    <div class="remark-footer unselectable">
        <button class="remark-save btn-link">保存</button>
        <button class="remark-cancel btn-link">取消</button>
    </div>

        <!-- clone the template $('.new-remark-reply').html() to here.-->
        <div class="remark-notice">在作者公开此批注前，只有你和作者可见。</div>
    </div>
</div>

<!-- template for new remark/reply -->
<div class="new-remark-reply side-remark-hidden">
    <div class="remark-head"><a><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/default-head.jpg"></a></div>
    <div class="remark-author unselectable"></div>
    <div class="remark-editor" contenteditable="true" spellcheck="false"></div>
    <!-- this will be filled up by js.
    <div class="inline-error">402/400</div> for new remark
    <div class="inline-error">202/200</div> for new reply
    -->
    <div class="remark-footer unselectable">
        <button class="remark-save btn-link">保存</button>
        <button class="remark-cancel btn-link">取消</button>
    </div>
</div>

<!-- template for .remark-item/.remark-reply -->
<div class="remark-item-reply side-remark-hidden">
    <div class="remark-head"><a><img src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/default-head.jpg"></a></div>
    <div class="remark-author unselectable"></div>
    <div class="remark-delete-link unselectable"><span class="icon-remove"></span></div> <!--This is mainly for deleting remark-reply, shown when author/remark hovering on remark-reply.-->
    <div class="remark-editor" contenteditable="true" spellcheck="false"></div>
    <!-- this will be filled up by js.
    <div class="inline-error">402/400</div> for new remark
    <div class="inline-error">202/200</div> for new reply
    -->
    <div class="remark-footer unselectable">
        <button class="remark-edit btn-link">修改</button>
        <button class="remark-save btn-link">保存</button>
        <button class="remark-cancel btn-link">取消</button>
        <button class="remark-delete btn-link">删除</button>
    </div>
</div>

<!-- template for remark-item-->
<div class="remark-item side-remark-hidden" data-rand-id="" data-version-id="">
    <div class="remark-published-link unselectable"><span class="icon-link icon-rotate-90"></span></div>
    <ul class="remark-options theme unselectable theme-white">
        <li class="remark-private"><span class="icon-eye-close"></span><span>私有</span></li>
        <li class="remark-public"><span class="icon-group"></span><span>公开</span></li>
        <li class="remark-delete"><span class="icon-remove"></span><span>删除</span></li>
    </ul>

    <!-- clone the template $('.remark-item-reply').html() to here.-->

    <button class="remark-reply-view-more btn-link">查看更早的 5 条回复</button>
    <div class="remark-replies">
        <!--
        <div class="remark-reply">
            clone the template $('.remark-item-reply').html() to here.
        </div>
        -->
    </div>

    <div class="leave-reply unselectable"><span>回复批注</span></div>
    <div class="new-reply">
        <!-- clone the template $('.new-remark-reply').html() to here.-->
    </div>
</div>

<!-- jiawzhang NOTICE: .remark-icons will be put to mdeditor.mako and user_note.mako, where next to .wmd-preview -->
<!-- <div class="remark-icons"></div> -->

<!-- template for remark-icon -->
<div class="remark-icon unselectable side-remark-hidden remark-icon-empty" style="display: none;">
    <span class="icon-stack">
        <i class="glyph-comment"></i>
        <span class="remark-count"></span>
    </span>
</div>


<!-- canvas, hidden always, this is used to convert svg to canvas and then convert canvas to png. -->
<canvas id="svg-canvas-image" class="editor-reader-hidden-always"></canvas>

<!-- This is the image panel to hold enlarged image/svg. -->
<div id="large-image-panel">
    <img id="large-image">
</div>


    


    <!-- Hidden Popup Modal -->
    <div id="notification-popup-window" class="modal hide fade theme theme-white" tabindex="-1" role="dialog" aria-labelledby="notification-title" aria-hidden="true">
        <div class="modal-header theme theme-white">
            <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>
            <h3 id="notification-title">通知</h3>
        </div>
        <div class="modal-body theme theme-white">
            <p></p>
        </div>
        <div class="modal-footer theme theme-white">
            <button id="notification-cancel" class="btn" data-dismiss="modal" aria-hidden="true">取消</button>
            <button id="notification-confirm" class="btn btn-primary">确认</button>
        </div>
    </div>

    <!-- zybuluo's foot -->

    <script src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/288313bb.base.lib.min.js"></script>

    <script>
        Namespace('com.zybuluo.base');
        com.zybuluo.base.initData = {
            globalPromptUrl: "https://www.zybuluo.com/global/prompt",
        };
    </script>

    
    <!--mathjax-->
    <!--blacker: 1 below means font weight.-->
    <script type="text/x-mathjax-config;executed=true">
        MathJax.Hub.Config({ tex2jax: { inlineMath: [['$','$'], ["\\(","\\)"]], processEscapes: true }, TeX: { equationNumbers: { autoNumber: "AMS" } }, messageStyle: "none", SVG: { blacker: 1 }});
    </script>
    <script src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/mathJax.js"></script>
    <!--mathjax source code is here: https://github.com/mathjax/MathJax.-->
    <script src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/mathJax(1).js"></script>

    <script>
        Namespace('com.zybuluo.mdeditor.layout');
        com.zybuluo.mdeditor.layout.initData = {
            // '' means not logged in, otherwise the logged in username, for mdeditor.mako, this value will be reset in render.js otherwise, for user_note.mako, it's rendered by server side.
            loggedInUsername: 'markliu2019',
            isPageOwner: 'False' === 'True' ? true : false,
            loginComeFromUrl: 'https://www.zybuluo.com/login?return_to=https%3A%2F%2Fwww.zybuluo.com%2Fhanbingtao%2Fnote%2F476663',
            noteRemarksUrl: "https://www.zybuluo.com/note/476663/remarks", 
            newNoteRemarkUrl: "https://www.zybuluo.com/note/476663/remark/new", 
            updateNoteRemarkUrl: "https://www.zybuluo.com/note/476663/remark/update", 
            deleteNoteRemarkUrl: "https://www.zybuluo.com/note/476663/remark/delete", 
            publishNoteRemarkUrl: "https://www.zybuluo.com/note/476663/remark/publish", 
            newNoteRemarkReplyUrl: "https://www.zybuluo.com/note/476663/remark_reply/new", 
            updateNoteRemarkReplyUrl: "https://www.zybuluo.com/note/476663/remark_reply/update", 
            deleteNoteRemarkReplyUrl: "https://www.zybuluo.com/note/476663/remark_reply/delete", 
        };

        // BEGIN: pace.js configuration
        window.paceOptions = {
            // disable others, enable for ajax call only,
            ajax: true,
            document: false,
            elements: false,
            eventLag: false,
        };
        // jiawzhang NOTICE: to make sure pace.js is working for any ajax call especially the jquery ajax, add 'Pace.restart()' into jquery ajax call like '$.post'
        // Originally, pace 0.5.6 doesn't support jquery ajax, see details in: https://github.com/HubSpot/pace/issues/29
        // END: pace.js configuration

    </script>

    <script src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/7a70106e.layout.lib.min.js"></script>

    <script src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/dc648f35.layout.min.js"></script><div id="medium-editor-toolbar-1" class="medium-editor-toolbar" style=""><ul id="medium-editor-toolbar-actions" class="medium-editor-toolbar-actions clearfix"><li><button data-action="remark" class=" medium-editor-button-first"><i class="icon-comment"></i></button></li><li><button data-action="highlight" class=" medium-editor-button-last"><i class="icon-pencil"></i></button></li></ul></div>



    

    <!-- https://www.zybuluo.com/static/assets/mdeditor/user_note.lib.min.js -->
    <!-- -->

    <script>
        Namespace('com.zybuluo.mdeditor.user_note');
        com.zybuluo.mdeditor.user_note.initData = {
            isLoggedIn: 'True',
            mdeditorUrl: "https://www.zybuluo.com/mdeditor",
            passwordPassed: 'True' === 'True' ? true : false,
        };
    </script>

    <script src="./零基础入门深度学习(3) - 神经网络和反向传播算法 - 作业部落 Cmd Markdown 编辑阅读器_files/6cd3112e.user_note.min.js"></script>





    
</body></html>